From 51e9b60f182f1d7fea13a1e8bef52fcc77649217 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 3 Feb 2024 20:23:55 -0500 Subject: [PATCH 001/171] let there be presets --- bbot/core/preset/__init__.py | 0 bbot/core/preset/args.py | 256 +++++++++++++++++++++++++++++++++++ bbot/core/preset/files.py | 36 +++++ bbot/core/preset/preset.py | 34 +++++ 4 files changed, 326 insertions(+) create mode 100644 bbot/core/preset/__init__.py create mode 100644 bbot/core/preset/args.py create mode 100644 bbot/core/preset/files.py create mode 100644 bbot/core/preset/preset.py diff --git a/bbot/core/preset/__init__.py b/bbot/core/preset/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bbot/core/preset/args.py b/bbot/core/preset/args.py new file mode 100644 index 000000000..c03647c5f --- /dev/null +++ b/bbot/core/preset/args.py @@ -0,0 +1,256 @@ +import sys +import argparse +from pathlib import Path +from omegaconf import OmegaConf +from contextlib import suppress + +from ..helpers.logger import log_to_stderr +from ..helpers.misc import chain_lists, match_and_exit, is_file + + +class BBOTArgumentParser(argparse.ArgumentParser): + _dummy = False + + def __init__(self, *args, **kwargs): + if not self._dummy: + module_loader = kwargs.pop("_module_loader") + self._module_choices = sorted(set(module_loader.configs(type="scan"))) + self._output_module_choices = sorted(set(module_loader.configs(type="output"))) + self._flag_choices = set() + for m, c in module_loader.preloaded().items(): + self._flag_choices.update(set(c.get("flags", []))) + + def parse_args(self, *args, **kwargs): + """ + Allow space or comma-separated entries for modules and targets + For targets, also allow input files containing additional targets + """ + ret = super().parse_args(*args, **kwargs) + # silent implies -y + if ret.silent: + ret.yes = True + ret.modules = chain_lists(ret.modules) + ret.exclude_modules = chain_lists(ret.exclude_modules) + ret.output_modules = chain_lists(ret.output_modules) + ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}") + ret.whitelist = chain_lists(ret.whitelist, try_files=True, msg="Reading whitelist from file: {filename}") + ret.blacklist = chain_lists(ret.blacklist, try_files=True, msg="Reading blacklist from file: {filename}") + ret.flags = chain_lists(ret.flags) + ret.exclude_flags = chain_lists(ret.exclude_flags) + ret.require_flags = chain_lists(ret.require_flags) + if not self._dummy: + for m in ret.modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in ret.exclude_modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in ret.output_modules: + if m not in self._output_module_choices: + match_and_exit(m, self._output_module_choices, msg="output module") + for f in set(ret.flags + ret.require_flags): + if f not in self._flag_choices: + match_and_exit(f, self._flag_choices, msg="flag") + return ret + + +class DummyArgumentParser(BBOTArgumentParser): + _dummy = True + + def error(self, message): + pass + + +scan_examples = [ + ( + "Subdomains", + "Perform a full subdomain enumeration on evilcorp.com", + "bbot -t evilcorp.com -f subdomain-enum", + ), + ( + "Subdomains (passive only)", + "Perform a passive-only subdomain enumeration on evilcorp.com", + "bbot -t evilcorp.com -f subdomain-enum -rf passive", + ), + ( + "Subdomains + port scan + web screenshots", + "Port-scan every subdomain, screenshot every webpage, output to current directory", + "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .", + ), + ( + "Subdomains + basic web scan", + "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules", + "bbot -t evilcorp.com -f subdomain-enum web-basic", + ), + ( + "Web spider", + "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.", + "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2", + ), + ( + "Everything everywhere all at once", + "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei", + "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly", + ), +] + +usage_examples = [ + ( + "List modules", + "", + "bbot -l", + ), + ( + "List flags", + "", + "bbot -lf", + ), +] + + +epilog = "EXAMPLES\n" +for example in (scan_examples, usage_examples): + for title, description, command in example: + epilog += f"\n {title}:\n {command}\n" + + +parser = BBOTArgumentParser( + description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog +) +dummy_parser = DummyArgumentParser( + description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog +) +for p in (parser, dummy_parser): + p.add_argument("--help-all", action="store_true", help="Display full help including module config options") + target = p.add_argument_group(title="Target") + target.add_argument("-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET") + target.add_argument( + "-w", + "--whitelist", + nargs="+", + default=[], + help="What's considered in-scope (by default it's the same as --targets)", + ) + target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things") + target.add_argument( + "--strict-scope", + action="store_true", + help="Don't consider subdomains of target/whitelist to be in-scope", + ) + modules = p.add_argument_group(title="Modules") + modules.add_argument( + "-m", + "--modules", + nargs="+", + default=[], + help=f'Modules to enable. Choices: {",".join(module_choices)}', + metavar="MODULE", + ) + modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") + modules.add_argument( + "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" + ) + modules.add_argument( + "-f", + "--flags", + nargs="+", + default=[], + help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}', + metavar="FLAG", + ) + modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") + modules.add_argument( + "-rf", + "--require-flags", + nargs="+", + default=[], + help=f"Only enable modules with these flags (e.g. -rf passive)", + metavar="FLAG", + ) + modules.add_argument( + "-ef", + "--exclude-flags", + nargs="+", + default=[], + help=f"Disable modules with these flags. (e.g. -ef aggressive)", + metavar="FLAG", + ) + modules.add_argument( + "-om", + "--output-modules", + nargs="+", + default=["human", "json", "csv"], + help=f'Output module(s). Choices: {",".join(output_module_choices)}', + metavar="MODULE", + ) + modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") + scan = p.add_argument_group(title="Scan") + scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") + scan.add_argument( + "-o", + "--output-dir", + metavar="DIR", + ) + scan.add_argument( + "-c", + "--config", + nargs="*", + help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", + metavar="CONFIG", + ) + scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") + scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") + scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") + scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") + scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") + scan.add_argument( + "--current-config", + action="store_true", + help="Show current config in YAML format", + ) + deps = p.add_argument_group( + title="Module dependencies", description="Control how modules install their dependencies" + ) + g2 = deps.add_mutually_exclusive_group() + g2.add_argument("--no-deps", action="store_true", help="Don't install module dependencies") + g2.add_argument("--force-deps", action="store_true", help="Force install all module dependencies") + g2.add_argument("--retry-deps", action="store_true", help="Try again to install failed module dependencies") + g2.add_argument( + "--ignore-failed-deps", action="store_true", help="Run modules even if they have failed dependencies" + ) + g2.add_argument("--install-all-deps", action="store_true", help="Install dependencies for all modules") + agent = p.add_argument_group(title="Agent", description="Report back to a central server") + agent.add_argument("-a", "--agent-mode", action="store_true", help="Start in agent mode") + misc = p.add_argument_group(title="Misc") + misc.add_argument("--version", action="store_true", help="show BBOT version and exit") + + +cli_options = None +with suppress(Exception): + cli_options = dummy_parser.parse_args() + + +cli_config = [] + + +def get_config(): + global cli_config + with suppress(Exception): + if cli_options.config: + cli_config = cli_options.config + if cli_config: + filename = Path(cli_config[0]).resolve() + if len(cli_config) == 1 and is_file(filename): + try: + conf = OmegaConf.load(str(filename)) + log_to_stderr(f"Loaded custom config from {filename}") + return conf + except Exception as e: + log_to_stderr(f"Error parsing custom config at {filename}: {e}", level="ERROR") + sys.exit(2) + try: + return OmegaConf.from_cli(cli_config) + except Exception as e: + log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") + sys.exit(2) diff --git a/bbot/core/preset/files.py b/bbot/core/preset/files.py new file mode 100644 index 000000000..719997915 --- /dev/null +++ b/bbot/core/preset/files.py @@ -0,0 +1,36 @@ +import sys +from pathlib import Path +from omegaconf import OmegaConf + +from ..helpers.misc import mkdir +from ..errors import ConfigLoadError +from ..helpers.logger import log_to_stderr + +bbot_code_dir = Path(__file__).parent.parent.parent +config_dir = (Path.home() / ".config" / "bbot").resolve() +defaults_filename = (bbot_code_dir / "defaults.yml").resolve() +config_filename = (config_dir / "bbot.yml").resolve() +secrets_filename = (config_dir / "secrets.yml").resolve() + + +def _get_config(filename, name="config"): + filename = Path(filename).resolve() + try: + conf = OmegaConf.load(str(filename)) + cli_silent = any(x in sys.argv for x in ("-s", "--silent")) + if __name__ == "__main__" and not cli_silent: + log_to_stderr(f"Loaded {name} from {filename}") + return conf + except Exception as e: + if filename.exists(): + raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") + return OmegaConf.create() + + +def get_config(): + default_config = _get_config(defaults_filename, name="defaults") + return OmegaConf.merge( + default_config, + _get_config(config_filename, name="config"), + _get_config(secrets_filename, name="secrets"), + ) diff --git a/bbot/core/preset/preset.py b/bbot/core/preset/preset.py new file mode 100644 index 000000000..1a9c0f3d1 --- /dev/null +++ b/bbot/core/preset/preset.py @@ -0,0 +1,34 @@ +import omegaconf +from pathlib import Path +from omegaconf import OmegaConf + +from . import files +from ..helpers.misc import mkdir + + +class Preset(omegaconf.dictconfig.DictConfig): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # first, we load config files + # this populates defaults and necessary stuff like: + # - bbot home directory + # - module load directories (needed for preloading modules) + self.load_config_files() + + # next, we load environment variables + # todo: automatically propagate config values to environ? (would require __setitem__ hooks) + # self.load_environ() + + # next, we load module defaults + # this populates valid modules + flags (needed for parsing CLI args) + # self.load_module_configs() + + # finally, we parse CLI args + # self.parse_cli_args() + + def load_config_files(self): + self.update(files.get_config()) + + def load_cli_args(self): + pass From 718084e41dedf7af11440ced60238ae61236e727 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 1 Feb 2024 17:43:02 -0500 Subject: [PATCH 002/171] JSON module: option for SIEM-friendly output --- bbot/modules/output/json.py | 14 +++++++++++--- .../test_step_2/module_tests/test_module_json.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index f13cf7808..a380ac9a1 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -7,16 +7,24 @@ class JSON(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"} - options = {"output_file": "", "console": False} - options_desc = {"output_file": "Output to file", "console": "Output to console"} + options = {"output_file": "", "console": False, "siem_friendly": False} + options_desc = { + "output_file": "Output to file", + "console": "Output to console", + "siem_friendly": "Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc.", + } _preserve_graph = True async def setup(self): self._prep_output_dir("output.ndjson") + self.siem_friendly = self.config.get("siem_friendly", False) return True async def handle_event(self, event): - event_str = json.dumps(dict(event)) + event_json = dict(event) + if self.siem_friendly: + event_json["data"] = {event.type: event_json.pop("data", "")} + event_str = json.dumps(event_json) if self.file is not None: self.file.write(event_str + "\n") self.file.flush() diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 6dafb68a5..1e67db085 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -12,3 +12,18 @@ def check(self, module_test, events): e = event_from_json(json.loads(lines[0])) assert e.type == "SCAN" assert e.data == f"{module_test.scan.name} ({module_test.scan.id})" + + +class TestJSONSIEMFriendly(ModuleTestBase): + modules_overrides = ["json"] + config_overrides = {"output_modules": {"json": {"siem_friendly": True}}} + + def check(self, module_test, events): + txt_file = module_test.scan.home / "output.ndjson" + lines = list(module_test.scan.helpers.read_file(txt_file)) + passed = False + for line in lines: + e = json.loads(line) + if e["data"] == {"DNS_NAME": "blacklanternsecurity.com"}: + passed = True + assert passed From c0c75597b437abbd8cf57237047bbd1e06ca1c59 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 1 Feb 2024 17:50:24 -0500 Subject: [PATCH 003/171] updated docs --- docs/scanning/tips_and_tricks.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md index 0572bedb2..1ed370e52 100644 --- a/docs/scanning/tips_and_tricks.md +++ b/docs/scanning/tips_and_tricks.md @@ -53,6 +53,24 @@ You can also pair the web spider with subdomain enumeration: bbot -t evilcorp.com -f subdomain-enum -c spider.yml ``` +### Ingesting BBOT Data Into SIEM (Elastic, Splunk) + +If your goal is to feed BBOT data into a SIEM such as Elastic, make sure to enable this option when scanning: + +```bash +bbot -t evilcorp.com -c output_modules.json.siem_friendly=true +``` + +This nests the event's `.data` beneath its event type like so: +```json +{ + "type": "DNS_NAME", + "data": { + "DNS_NAME": "blacklanternsecurity.com" + } +} +``` + ### Custom HTTP Proxy Web pentesters may appreciate BBOT's ability to quickly populate Burp Suite site maps for all subdomains in a target. If your scan includes gowitness, this will capture the traffic as if you manually visited each website in your browser -- including auxiliary web resources and javascript API calls. To accomplish this, set the `http_proxy` config option like so: From 30e9b642671a0491d493a8746acd64cfb607e015 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 1 Feb 2024 17:52:28 -0500 Subject: [PATCH 004/171] updated docs --- docs/scanning/tips_and_tricks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md index 1ed370e52..5e115bcde 100644 --- a/docs/scanning/tips_and_tricks.md +++ b/docs/scanning/tips_and_tricks.md @@ -55,7 +55,7 @@ bbot -t evilcorp.com -f subdomain-enum -c spider.yml ### Ingesting BBOT Data Into SIEM (Elastic, Splunk) -If your goal is to feed BBOT data into a SIEM such as Elastic, make sure to enable this option when scanning: +If your goal is to feed BBOT data into a SIEM such as Elastic, be sure to enable this option when scanning: ```bash bbot -t evilcorp.com -c output_modules.json.siem_friendly=true From 688449074647223265554c6b81eb9688143d498d Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Fri, 2 Feb 2024 22:03:08 +0000 Subject: [PATCH 005/171] Refresh module docs --- docs/scanning/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 0ebfc89dd..2ed2d61de 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -353,6 +353,7 @@ Many modules accept their own configuration options. These options have the abil | output_modules.human.output_file | str | Output to file | | | output_modules.json.console | bool | Output to console | False | | output_modules.json.output_file | str | Output to file | | +| output_modules.json.siem_friendly | bool | Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc. | False | | output_modules.neo4j.password | str | Neo4j password | bbotislife | | output_modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 | | output_modules.neo4j.username | str | Neo4j username | neo4j | From 954f1ac9e821ab97921f77e8354b3cc9b6953f2f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 9 Feb 2024 15:40:10 -0500 Subject: [PATCH 006/171] better layout for bbot core --- bbot/__init__.py | 8 - bbot/cli.py | 366 +----------------- bbot/core/__init__.py | 5 +- bbot/core/{preset => config}/__init__.py | 0 bbot/core/config/args.py | 272 +++++++++++++ bbot/core/{configurator => config}/environ.py | 0 bbot/core/{preset => config}/files.py | 1 - bbot/core/config/logger.py | 226 +++++++++++ bbot/core/configurator/__init__.py | 103 ----- bbot/core/configurator/args.py | 255 ------------ bbot/core/configurator/files.py | 40 -- bbot/core/core.py | 63 +++ bbot/core/helpers/depsinstaller/installer.py | 13 +- bbot/core/logger/__init__.py | 10 - bbot/core/logger/logger.py | 238 ------------ bbot/core/{helpers => }/modules.py | 99 +++-- bbot/core/preset/args.py | 256 ------------ bbot/core/preset/preset.py | 34 -- bbot/defaults.yml | 2 + bbot/modules/__init__.py | 14 - bbot/scanner/scanner.py | 39 +- bbot/scripts/docs.py | 19 +- bbot/test/bbot_fixtures.py | 14 +- bbot/test/test_step_1/test__module__tests.py | 4 +- bbot/test/test_step_1/test_docs.py | 5 +- bbot/test/test_step_1/test_presets.py | 21 + bbot/test/test_step_2/module_tests/base.py | 5 +- 27 files changed, 701 insertions(+), 1411 deletions(-) rename bbot/core/{preset => config}/__init__.py (100%) create mode 100644 bbot/core/config/args.py rename bbot/core/{configurator => config}/environ.py (100%) rename bbot/core/{preset => config}/files.py (97%) create mode 100644 bbot/core/config/logger.py delete mode 100644 bbot/core/configurator/__init__.py delete mode 100644 bbot/core/configurator/args.py delete mode 100644 bbot/core/configurator/files.py create mode 100644 bbot/core/core.py delete mode 100644 bbot/core/logger/__init__.py delete mode 100644 bbot/core/logger/logger.py rename bbot/core/{helpers => }/modules.py (87%) delete mode 100644 bbot/core/preset/args.py delete mode 100644 bbot/core/preset/preset.py create mode 100644 bbot/test/test_step_1/test_presets.py diff --git a/bbot/__init__.py b/bbot/__init__.py index 1d95273e3..7c3310ef7 100644 --- a/bbot/__init__.py +++ b/bbot/__init__.py @@ -1,10 +1,2 @@ # version placeholder (replaced by poetry-dynamic-versioning) __version__ = "0.0.0" - -# global app config -from .core import configurator - -config = configurator.config - -# helpers -from .core import helpers diff --git a/bbot/cli.py b/bbot/cli.py index 1eaf66996..bf95a67cc 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,390 +1,36 @@ #!/usr/bin/env python3 -import os -import re import sys import asyncio import logging import traceback -from aioconsole import ainput -from omegaconf import OmegaConf -from contextlib import suppress +from bbot.core.helpers.logger import log_to_stderr # fix tee buffering sys.stdout.reconfigure(line_buffering=True) -# logging -from bbot.core.logger import get_log_level, toggle_log_level - -import bbot.core.errors -from bbot import __version__ -from bbot.modules import module_loader -from bbot.core.configurator.args import parser -from bbot.core.helpers.logger import log_to_stderr -from bbot.core.configurator import ensure_config_files, check_cli_args, environ - log = logging.getLogger("bbot.cli") sys.stdout.reconfigure(line_buffering=True) -log_level = get_log_level() - - -from . import config - - -err = False -scan_name = "" +from bbot.core import CORE async def _main(): - global err - global scan_name - environ.cli_execution = True - - # async def monitor_tasks(): - # in_row = 0 - # while 1: - # try: - # print('looooping') - # tasks = asyncio.all_tasks() - # current_task = asyncio.current_task() - # if len(tasks) == 1 and list(tasks)[0] == current_task: - # print('no tasks') - # in_row += 1 - # else: - # in_row = 0 - # for t in tasks: - # print(t) - # if in_row > 2: - # break - # await asyncio.sleep(1) - # except BaseException as e: - # print(traceback.format_exc()) - # with suppress(BaseException): - # await asyncio.sleep(.1) - - # monitor_tasks_task = asyncio.create_task(monitor_tasks()) - - ensure_config_files() - - try: - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - options = parser.parse_args() - check_cli_args() - - # --version - if options.version: - log.stdout(__version__) - sys.exit(0) - return - - # --current-config - if options.current_config: - log.stdout(f"{OmegaConf.to_yaml(config)}") - sys.exit(0) - return - - log.verbose(f'Command: {" ".join(sys.argv)}') - - if options.agent_mode: - from bbot.agent import Agent - - agent = Agent(config) - success = agent.setup() - if success: - await agent.start() - - else: - from bbot.scanner import Scanner - - try: - output_modules = set(options.output_modules) - module_filtering = False - if (options.list_modules or options.help_all) and not any([options.flags, options.modules]): - module_filtering = True - modules = set(module_loader.preloaded(type="scan")) - else: - modules = set(options.modules) - # enable modules by flags - for m, c in module_loader.preloaded().items(): - module_type = c.get("type", "scan") - if m not in modules: - flags = c.get("flags", []) - if "deadly" in flags: - continue - for f in options.flags: - if f in flags: - log.verbose(f'Enabling {m} because it has flag "{f}"') - if module_type == "output": - output_modules.add(m) - else: - modules.add(m) - - default_output_modules = ["human", "json", "csv"] - - # Make a list of the modules which can be output to the console - consoleable_output_modules = [ - k for k, v in module_loader.preloaded(type="output").items() if "console" in v["config"] - ] - - # if none of the output modules provided on the command line are consoleable, don't turn off the defaults. Instead, just add the one specified to the defaults. - if not any(o in consoleable_output_modules for o in output_modules): - output_modules.update(default_output_modules) - - scanner = Scanner( - *options.targets, - modules=list(modules), - output_modules=list(output_modules), - output_dir=options.output_dir, - config=config, - name=options.name, - whitelist=options.whitelist, - blacklist=options.blacklist, - strict_scope=options.strict_scope, - force_start=options.force, - ) - - if options.install_all_deps: - all_modules = list(module_loader.preloaded()) - scanner.helpers.depsinstaller.force_deps = True - succeeded, failed = await scanner.helpers.depsinstaller.install(*all_modules) - log.info("Finished installing module dependencies") - return False if failed else True - - scan_name = str(scanner.name) - - # enable modules by dependency - # this is only a basic surface-level check - # todo: recursive dependency graph with networkx or topological sort? - all_modules = list(set(scanner._scan_modules + scanner._internal_modules + scanner._output_modules)) - while 1: - changed = False - dep_choices = module_loader.recommend_dependencies(all_modules) - if not dep_choices: - break - for event_type, deps in dep_choices.items(): - if event_type in ("*", "all"): - continue - # skip resolving dependency if a target provides the missing type - if any(e.type == event_type for e in scanner.target.events): - continue - required_by = deps.get("required_by", []) - recommended = deps.get("recommended", []) - if not recommended: - log.hugewarning( - f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no modules produce it" - ) - elif len(recommended) == 1: - log.verbose( - f"Enabling {next(iter(recommended))} because {len(required_by):,} modules ({','.join(required_by)}) rely on it for {event_type}" - ) - all_modules = list(set(all_modules + list(recommended))) - scanner._scan_modules = list(set(scanner._scan_modules + list(recommended))) - changed = True - else: - log.hugewarning( - f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no enabled module produces it" - ) - log.hugewarning( - f"Recommend enabling one or more of the following modules which produce {event_type}:" - ) - for m in recommended: - log.warning(f" - {m}") - if not changed: - break - - # required flags - modules = set(scanner._scan_modules) - for m in scanner._scan_modules: - flags = module_loader._preloaded.get(m, {}).get("flags", []) - if not all(f in flags for f in options.require_flags): - log.verbose( - f"Removing {m} because it does not have the required flags: {'+'.join(options.require_flags)}" - ) - with suppress(KeyError): - modules.remove(m) - - # excluded flags - for m in scanner._scan_modules: - flags = module_loader._preloaded.get(m, {}).get("flags", []) - if any(f in flags for f in options.exclude_flags): - log.verbose(f"Removing {m} because of excluded flag: {','.join(options.exclude_flags)}") - with suppress(KeyError): - modules.remove(m) - - # excluded modules - for m in options.exclude_modules: - if m in modules: - log.verbose(f"Removing {m} because it is excluded") - with suppress(KeyError): - modules.remove(m) - scanner._scan_modules = list(modules) - - log_fn = log.info - if options.list_modules or options.help_all: - log_fn = log.stdout - - help_modules = list(modules) - if module_filtering: - help_modules = None - - if options.help_all: - log_fn(parser.format_help()) - - if options.list_flags: - log.stdout("") - log.stdout("### FLAGS ###") - log.stdout("") - for row in module_loader.flags_table(flags=options.flags).splitlines(): - log.stdout(row) - return - - log_fn("") - log_fn("### MODULES ###") - log_fn("") - for row in module_loader.modules_table(modules=help_modules).splitlines(): - log_fn(row) - - if options.help_all: - log_fn("") - log_fn("### MODULE OPTIONS ###") - log_fn("") - for row in module_loader.modules_options_table(modules=help_modules).splitlines(): - log_fn(row) - - if options.list_modules or options.list_flags or options.help_all: - return - - module_list = module_loader.filter_modules(modules=modules) - deadly_modules = [] - active_modules = [] - active_aggressive_modules = [] - slow_modules = [] - for m in module_list: - if m[0] in scanner._scan_modules: - if "deadly" in m[-1]["flags"]: - deadly_modules.append(m[0]) - if "active" in m[-1]["flags"]: - active_modules.append(m[0]) - if "aggressive" in m[-1]["flags"]: - active_aggressive_modules.append(m[0]) - if "slow" in m[-1]["flags"]: - slow_modules.append(m[0]) - if scanner._scan_modules: - if deadly_modules and not options.allow_deadly: - log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") - log.hugewarning(f"Deadly modules are highly intrusive") - log.hugewarning(f"Please specify --allow-deadly to continue") - return False - if active_modules: - if active_modules: - if active_aggressive_modules: - log.hugewarning( - "This is an (aggressive) active scan! Intrusive connections will be made to target" - ) - else: - log.hugewarning( - "This is a (safe) active scan. Non-intrusive connections will be made to target" - ) - else: - log.hugeinfo("This is a passive scan. No connections will be made to target") - if slow_modules: - log.warning( - f"You have enabled the following slow modules: {','.join(slow_modules)}. Scan may take a while" - ) - - scanner.helpers.word_cloud.load() - - await scanner._prep() - - if not options.dry_run: - log.trace(f"Command: {' '.join(sys.argv)}") - if not options.agent_mode and not options.yes and sys.stdin.isatty(): - log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}") - input() - - def handle_keyboard_input(keyboard_input): - kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") - if keyboard_input: - log.verbose(f'Got keyboard input: "{keyboard_input}"') - kill_match = kill_regex.match(keyboard_input) - if kill_match: - module = kill_match.group("module") - if module in scanner.modules: - log.hugewarning(f'Killing module: "{module}"') - scanner.manager.kill_module(module, message="killed by user") - else: - log.warning(f'Invalid module: "{module}"') - else: - toggle_log_level(logger=log) - scanner.manager.modules_status(_log=True) - - async def akeyboard_listen(): - allowed_errors = 10 - while 1: - keyboard_input = "a" - try: - keyboard_input = await ainput() - except Exception: - allowed_errors -= 1 - handle_keyboard_input(keyboard_input) - if allowed_errors <= 0: - break - - try: - keyboard_listen_task = asyncio.create_task(akeyboard_listen()) - - await scanner.async_start_without_generator() - finally: - keyboard_listen_task.cancel() - with suppress(asyncio.CancelledError): - await keyboard_listen_task - - except bbot.core.errors.ScanError as e: - log_to_stderr(str(e), level="ERROR") - except Exception: - raise - - except bbot.core.errors.BBOTError as e: - log_to_stderr(f"{e} (--debug for details)", level="ERROR") - if log_level <= logging.DEBUG: - log_to_stderr(traceback.format_exc(), level="DEBUG") - err = True - - except Exception: - log_to_stderr(f"Encountered unknown error: {traceback.format_exc()}", level="ERROR") - err = True - - finally: - # save word cloud - with suppress(BaseException): - save_success, filename = scanner.helpers.word_cloud.save() - if save_success: - log_to_stderr(f"Saved word cloud ({len(scanner.helpers.word_cloud):,} words) to {filename}") - # remove output directory if empty - with suppress(BaseException): - scanner.home.rmdir() - if err: - os._exit(1) + CORE.args + CORE.module_loader.preloaded() def main(): - global scan_name try: asyncio.run(_main()) except asyncio.CancelledError: - if get_log_level() <= logging.DEBUG: + if CORE.logger.log_level <= logging.DEBUG: log_to_stderr(traceback.format_exc(), level="DEBUG") except KeyboardInterrupt: msg = "Interrupted" - if scan_name: - msg = f"You killed {scan_name}" log_to_stderr(msg, level="WARNING") - if get_log_level() <= logging.DEBUG: + if CORE.logger.log_level <= logging.DEBUG: log_to_stderr(traceback.format_exc(), level="DEBUG") exit(1) diff --git a/bbot/core/__init__.py b/bbot/core/__init__.py index 52cf06cc5..6cfaecf0f 100644 --- a/bbot/core/__init__.py +++ b/bbot/core/__init__.py @@ -1,4 +1,3 @@ -# logging -from .logger import init_logging +from .core import BBOTCore -init_logging() +CORE = BBOTCore() diff --git a/bbot/core/preset/__init__.py b/bbot/core/config/__init__.py similarity index 100% rename from bbot/core/preset/__init__.py rename to bbot/core/config/__init__.py diff --git a/bbot/core/config/args.py b/bbot/core/config/args.py new file mode 100644 index 000000000..e8ed21af5 --- /dev/null +++ b/bbot/core/config/args.py @@ -0,0 +1,272 @@ +import sys +import argparse +from pathlib import Path +from omegaconf import OmegaConf +from contextlib import suppress + +from ..helpers.logger import log_to_stderr +from ..helpers.misc import chain_lists, match_and_exit, is_file + + +class BBOTArgumentParser(argparse.ArgumentParser): + """ + A subclass of argparse.ArgumentParser with several extra features: + - permissive parsing of modules/flags, allowing either space or comma-separated entries + - loading targets / config from files + - option to *not* exit on error + """ + + def __init__(self, *args, **kwargs): + self._dummy = kwargs.pop("_dummy", False) + self.bbot_core = kwargs.pop("_core") + self._module_choices = sorted(set(self.bbot_core.module_loader.configs(type="scan"))) + self._output_module_choices = sorted(set(self.bbot_core.module_loader.configs(type="output"))) + self._flag_choices = set() + for m, c in self.bbot_core.module_loader.preloaded().items(): + self._flag_choices.update(set(c.get("flags", []))) + super().__init__(*args, **kwargs) + + def error(self, message): + if not self._dummy: + return super().error(message) + + def parse_args(self, *args, **kwargs): + """ + Allow space or comma-separated entries for modules and targets + For targets, also allow input files containing additional targets + """ + ret = super().parse_args(*args, **kwargs) + # silent implies -y + if ret.silent: + ret.yes = True + ret.modules = chain_lists(ret.modules) + ret.exclude_modules = chain_lists(ret.exclude_modules) + ret.output_modules = chain_lists(ret.output_modules) + ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}") + ret.whitelist = chain_lists(ret.whitelist, try_files=True, msg="Reading whitelist from file: {filename}") + ret.blacklist = chain_lists(ret.blacklist, try_files=True, msg="Reading blacklist from file: {filename}") + ret.flags = chain_lists(ret.flags) + ret.exclude_flags = chain_lists(ret.exclude_flags) + ret.require_flags = chain_lists(ret.require_flags) + if not self._dummy: + for m in ret.modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in ret.exclude_modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in ret.output_modules: + if m not in self._output_module_choices: + match_and_exit(m, self._output_module_choices, msg="output module") + for f in set(ret.flags + ret.require_flags): + if f not in self._flag_choices: + match_and_exit(f, self._flag_choices, msg="flag") + return ret + + +class BBOTArgs: + scan_examples = [ + ( + "Subdomains", + "Perform a full subdomain enumeration on evilcorp.com", + "bbot -t evilcorp.com -f subdomain-enum", + ), + ( + "Subdomains (passive only)", + "Perform a passive-only subdomain enumeration on evilcorp.com", + "bbot -t evilcorp.com -f subdomain-enum -rf passive", + ), + ( + "Subdomains + port scan + web screenshots", + "Port-scan every subdomain, screenshot every webpage, output to current directory", + "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .", + ), + ( + "Subdomains + basic web scan", + "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules", + "bbot -t evilcorp.com -f subdomain-enum web-basic", + ), + ( + "Web spider", + "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.", + "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2", + ), + ( + "Everything everywhere all at once", + "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei", + "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly", + ), + ] + + usage_examples = [ + ( + "List modules", + "", + "bbot -l", + ), + ( + "List flags", + "", + "bbot -lf", + ), + ] + + epilog = "EXAMPLES\n" + for example in (scan_examples, usage_examples): + for title, description, command in example: + epilog += f"\n {title}:\n {command}\n" + + def __init__(self, core): + self.core = core + self.parser = self.create_parser(_core=self.core) + self.dummy_parser = self.create_parser(_core=self.core, _dummy=True) + self._parsed = None + self._cli_config = None + + @property + def parsed(self): + """ + Returns the parsed BBOT Argument Parser. + """ + if self._parsed is None: + self._parsed = self.dummy_parser.parse_args() + return self._parsed + + @property + def cli_config(self): + if self._cli_config is None: + with suppress(Exception): + if self.parsed.config: + cli_config = self.parsed.config + if cli_config: + filename = Path(cli_config[0]).resolve() + if len(cli_config) == 1 and is_file(filename): + try: + conf = OmegaConf.load(str(filename)) + log_to_stderr(f"Loaded custom config from {filename}") + return conf + except Exception as e: + log_to_stderr(f"Error parsing custom config at {filename}: {e}", level="ERROR") + sys.exit(2) + try: + self._cli_config = OmegaConf.from_cli(cli_config) + except Exception as e: + log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") + sys.exit(2) + if self._cli_config is None: + self._cli_config = OmegaConf.create({}) + return self._cli_config + + def create_parser(self, *args, **kwargs): + kwargs.update( + dict( + description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=self.epilog + ) + ) + p = BBOTArgumentParser(*args, **kwargs) + p.add_argument("--help-all", action="store_true", help="Display full help including module config options") + target = p.add_argument_group(title="Target") + target.add_argument( + "-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET" + ) + target.add_argument( + "-w", + "--whitelist", + nargs="+", + default=[], + help="What's considered in-scope (by default it's the same as --targets)", + ) + target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things") + target.add_argument( + "--strict-scope", + action="store_true", + help="Don't consider subdomains of target/whitelist to be in-scope", + ) + modules = p.add_argument_group(title="Modules") + modules.add_argument( + "-m", + "--modules", + nargs="+", + default=[], + help=f'Modules to enable. Choices: {",".join(p._module_choices)}', + metavar="MODULE", + ) + modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") + modules.add_argument( + "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" + ) + modules.add_argument( + "-f", + "--flags", + nargs="+", + default=[], + help=f'Enable modules by flag. Choices: {",".join(sorted(p._flag_choices))}', + metavar="FLAG", + ) + modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") + modules.add_argument( + "-rf", + "--require-flags", + nargs="+", + default=[], + help=f"Only enable modules with these flags (e.g. -rf passive)", + metavar="FLAG", + ) + modules.add_argument( + "-ef", + "--exclude-flags", + nargs="+", + default=[], + help=f"Disable modules with these flags. (e.g. -ef aggressive)", + metavar="FLAG", + ) + modules.add_argument( + "-om", + "--output-modules", + nargs="+", + default=["human", "json", "csv"], + help=f'Output module(s). Choices: {",".join(p._output_module_choices)}', + metavar="MODULE", + ) + modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") + scan = p.add_argument_group(title="Scan") + scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") + scan.add_argument( + "-o", + "--output-dir", + metavar="DIR", + ) + scan.add_argument( + "-c", + "--config", + nargs="*", + help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", + metavar="CONFIG", + ) + scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") + scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") + scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") + scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") + scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") + scan.add_argument( + "--current-config", + action="store_true", + help="Show current config in YAML format", + ) + deps = p.add_argument_group( + title="Module dependencies", description="Control how modules install their dependencies" + ) + g2 = deps.add_mutually_exclusive_group() + g2.add_argument("--no-deps", action="store_true", help="Don't install module dependencies") + g2.add_argument("--force-deps", action="store_true", help="Force install all module dependencies") + g2.add_argument("--retry-deps", action="store_true", help="Try again to install failed module dependencies") + g2.add_argument( + "--ignore-failed-deps", action="store_true", help="Run modules even if they have failed dependencies" + ) + g2.add_argument("--install-all-deps", action="store_true", help="Install dependencies for all modules") + agent = p.add_argument_group(title="Agent", description="Report back to a central server") + agent.add_argument("-a", "--agent-mode", action="store_true", help="Start in agent mode") + misc = p.add_argument_group(title="Misc") + misc.add_argument("--version", action="store_true", help="show BBOT version and exit") + return p diff --git a/bbot/core/configurator/environ.py b/bbot/core/config/environ.py similarity index 100% rename from bbot/core/configurator/environ.py rename to bbot/core/config/environ.py diff --git a/bbot/core/preset/files.py b/bbot/core/config/files.py similarity index 97% rename from bbot/core/preset/files.py rename to bbot/core/config/files.py index 719997915..af1549253 100644 --- a/bbot/core/preset/files.py +++ b/bbot/core/config/files.py @@ -2,7 +2,6 @@ from pathlib import Path from omegaconf import OmegaConf -from ..helpers.misc import mkdir from ..errors import ConfigLoadError from ..helpers.logger import log_to_stderr diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py new file mode 100644 index 000000000..5a9f9bdf0 --- /dev/null +++ b/bbot/core/config/logger.py @@ -0,0 +1,226 @@ +import os +import sys +import logging +from copy import copy +import logging.handlers +from pathlib import Path + +from ..helpers.misc import mkdir, error_and_exit +from ..helpers.logger import colorize, loglevel_mapping + + +debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s") + + +class ColoredFormatter(logging.Formatter): + """ + Pretty colors for terminal + """ + + formatter = logging.Formatter("%(levelname)s %(message)s") + module_formatter = logging.Formatter("%(levelname)s %(name)s: %(message)s") + + def format(self, record): + colored_record = copy(record) + levelname = colored_record.levelname + levelshort = loglevel_mapping.get(levelname, "INFO") + colored_record.levelname = colorize(f"[{levelshort}]", level=levelname) + if levelname == "CRITICAL" or levelname.startswith("HUGE"): + colored_record.msg = colorize(colored_record.msg, level=levelname) + # remove name + if colored_record.name.startswith("bbot.modules."): + colored_record.name = colored_record.name.split("bbot.modules.")[-1] + return self.module_formatter.format(colored_record) + return self.formatter.format(colored_record) + + +class BBOTLogger: + + def __init__(self, core): + # custom logging levels + if getattr(logging, "STDOUT", None) is None: + self.addLoggingLevel("STDOUT", 100) + self.addLoggingLevel("TRACE", 49) + self.addLoggingLevel("HUGEWARNING", 31) + self.addLoggingLevel("HUGESUCCESS", 26) + self.addLoggingLevel("SUCCESS", 25) + self.addLoggingLevel("HUGEINFO", 21) + self.addLoggingLevel("HUGEVERBOSE", 16) + self.addLoggingLevel("VERBOSE", 15) + self.verbosity_levels_toggle = [logging.INFO, logging.VERBOSE, logging.DEBUG] + + self._loggers = None + self._log_handlers = None + self._log_level_override = None + self.core_logger = logging.getLogger("bbot") + self.core = core + + # Don't do this more than once + if len(self.core_logger.handlers) == 0: + for logger in self.loggers: + self.include_logger(logger) + + def addLoggingLevel(self, levelName, levelNum, methodName=None): + """ + Comprehensively adds a new logging level to the `logging` module and the + currently configured logging class. + + `levelName` becomes an attribute of the `logging` module with the value + `levelNum`. `methodName` becomes a convenience method for both `logging` + itself and the class returned by `logging.getLoggerClass()` (usually just + `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is + used. + + To avoid accidental clobberings of existing attributes, this method will + raise an `AttributeError` if the level name is already an attribute of the + `logging` module or if the method name is already present + + Example + ------- + >>> addLoggingLevel('TRACE', logging.DEBUG - 5) + >>> logging.getLogger(__name__).setLevel('TRACE') + >>> logging.getLogger(__name__).trace('that worked') + >>> logging.trace('so did this') + >>> logging.TRACE + 5 + + """ + if not methodName: + methodName = levelName.lower() + + if hasattr(logging, levelName): + raise AttributeError(f"{levelName} already defined in logging module") + if hasattr(logging, methodName): + raise AttributeError(f"{methodName} already defined in logging module") + if hasattr(logging.getLoggerClass(), methodName): + raise AttributeError(f"{methodName} already defined in logger class") + + # This method was inspired by the answers to Stack Overflow post + # http://stackoverflow.com/q/2183233/2988730, especially + # http://stackoverflow.com/a/13638084/2988730 + def logForLevel(self, message, *args, **kwargs): + if self.isEnabledFor(levelNum): + self._log(levelNum, message, args, **kwargs) + + def logToRoot(message, *args, **kwargs): + logging.log(levelNum, message, *args, **kwargs) + + logging.addLevelName(levelNum, levelName) + setattr(logging, levelName, levelNum) + setattr(logging.getLoggerClass(), methodName, logForLevel) + setattr(logging, methodName, logToRoot) + + @property + def loggers(self): + if self._loggers is None: + self._loggers = [ + logging.getLogger("bbot"), + logging.getLogger("asyncio"), + ] + return self._loggers + + def add_log_handler(self, handler, formatter=None): + if handler.formatter is None: + handler.setFormatter(debug_format) + for logger in self.loggers: + if handler not in logger.handlers: + logger.addHandler(handler) + + def remove_log_handler(self, handler): + for logger in self.loggers: + if handler in logger.handlers: + logger.removeHandler(handler) + + def include_logger(self, logger): + if logger not in self.loggers: + self.loggers.append(logger) + logger.setLevel(self.log_level) + for handler in self.log_handlers.values(): + logger.addHandler(handler) + + @property + def log_handlers(self): + if self._log_handlers is None: + log_dir = Path(self.core.config["home"]) / "logs" + if not mkdir(log_dir, raise_error=False): + error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})") + + # Main log file + main_handler = logging.handlers.TimedRotatingFileHandler( + f"{log_dir}/bbot.log", when="d", interval=1, backupCount=14 + ) + + # Separate log file for debugging + debug_handler = logging.handlers.TimedRotatingFileHandler( + f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14 + ) + + def stderr_filter(record): + if record.levelno == logging.STDOUT or ( + record.levelno == logging.TRACE and self.log_level > logging.DEBUG + ): + return False + if record.levelno < self.log_level: + return False + return True + + # Log to stderr + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(stderr_filter) + # Log to stdout + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(lambda x: x.levelno == logging.STDOUT) + # log to files + debug_handler.addFilter( + lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE and x.levelno != logging.STDOUT) + ) + main_handler.addFilter( + lambda x: x.levelno not in (logging.STDOUT, logging.TRACE) and x.levelno >= logging.VERBOSE + ) + + # Set log format + debug_handler.setFormatter(debug_format) + main_handler.setFormatter(debug_format) + stderr_handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s: %(message)s")) + stdout_handler.setFormatter(logging.Formatter("%(message)s")) + + self._log_handlers = { + "stderr": stderr_handler, + "stdout": stdout_handler, + "file_debug": debug_handler, + "file_main": main_handler, + } + return self._log_handlers + + @property + def log_level(self): + if self._log_level_override is not None: + return self._log_level_override + + if self.core.config.get("debug", False) or os.environ.get("BBOT_DEBUG", "").lower() in ("true", "yes"): + return logging.DEBUG + + loglevel = logging.INFO + if self.core.args.parsed.verbose: + loglevel = logging.VERBOSE + if self.core.args.parsed.debug: + loglevel = logging.DEBUG + return loglevel + + def set_log_level(self, level, logger=None): + if logger is not None: + logger.hugeinfo(f"Setting log level to {logging.getLevelName(level)}") + self.core.config["silent"] = False + self._log_level_override = level + for logger in self.loggers: + logger.setLevel(level) + + def toggle_log_level(self, logger=None): + if self.log_level in self.verbosity_levels_toggle: + for i, level in enumerate(self.verbosity_levels_toggle): + if self.log_level == level: + self.set_log_level( + self.verbosity_levels_toggle[(i + 1) % len(self.verbosity_levels_toggle)], logger=logger + ) + else: + self.set_log_level(self.verbosity_levels_toggle[0], logger=logger) diff --git a/bbot/core/configurator/__init__.py b/bbot/core/configurator/__init__.py deleted file mode 100644 index 15962ce59..000000000 --- a/bbot/core/configurator/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -import re -from omegaconf import OmegaConf - -from . import files, args, environ -from ..errors import ConfigLoadError -from ...modules import module_loader -from ..helpers.logger import log_to_stderr -from ..helpers.misc import error_and_exit, filter_dict, clean_dict, match_and_exit, is_file - -# cached sudo password -bbot_sudo_pass = None - -modules_config = OmegaConf.create( - { - "modules": module_loader.configs(type="scan"), - "output_modules": module_loader.configs(type="output"), - "internal_modules": module_loader.configs(type="internal"), - } -) - -try: - config = OmegaConf.merge( - # first, pull module defaults - modules_config, - # then look in .yaml files - files.get_config(), - # finally, pull from CLI arguments - args.get_config(), - ) -except ConfigLoadError as e: - error_and_exit(e) - - -config = environ.prepare_environment(config) -default_config = OmegaConf.merge(files.default_config, modules_config) - - -sentinel = object() - - -exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$") - - -def check_cli_args(): - conf = [a for a in args.cli_config if not is_file(a)] - all_options = None - for c in conf: - c = c.split("=")[0].strip() - v = OmegaConf.select(default_config, c, default=sentinel) - # if option isn't in the default config - if v is sentinel: - if exclude_from_validation.match(c): - continue - if all_options is None: - from ...modules import module_loader - - modules_options = set() - for module_options in module_loader.modules_options().values(): - modules_options.update(set(o[0] for o in module_options)) - global_options = set(default_config.keys()) - {"modules", "output_modules"} - all_options = global_options.union(modules_options) - match_and_exit(c, all_options, msg="module option") - - -def ensure_config_files(): - secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] - exclude_keys = ["modules", "output_modules", "internal_modules"] - - comment_notice = ( - "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" - + "# Please be sure to uncomment when inserting API keys, etc.\n" - ) - - # ensure bbot.yml - if not files.config_filename.exists(): - log_to_stderr(f"Creating BBOT config at {files.config_filename}") - no_secrets_config = OmegaConf.to_object(default_config) - no_secrets_config = clean_dict( - no_secrets_config, - *secrets_strings, - fuzzy=True, - exclude_keys=exclude_keys, - ) - yaml = OmegaConf.to_yaml(no_secrets_config) - yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) - with open(str(files.config_filename), "w") as f: - f.write(yaml) - - # ensure secrets.yml - if not files.secrets_filename.exists(): - log_to_stderr(f"Creating BBOT secrets at {files.secrets_filename}") - secrets_only_config = OmegaConf.to_object(default_config) - secrets_only_config = filter_dict( - secrets_only_config, - *secrets_strings, - fuzzy=True, - exclude_keys=exclude_keys, - ) - yaml = OmegaConf.to_yaml(secrets_only_config) - yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) - with open(str(files.secrets_filename), "w") as f: - f.write(yaml) - files.secrets_filename.chmod(0o600) diff --git a/bbot/core/configurator/args.py b/bbot/core/configurator/args.py deleted file mode 100644 index 173583827..000000000 --- a/bbot/core/configurator/args.py +++ /dev/null @@ -1,255 +0,0 @@ -import sys -import argparse -from pathlib import Path -from omegaconf import OmegaConf -from contextlib import suppress - -from ...modules import module_loader -from ..helpers.logger import log_to_stderr -from ..helpers.misc import chain_lists, match_and_exit, is_file - -module_choices = sorted(set(module_loader.configs(type="scan"))) -output_module_choices = sorted(set(module_loader.configs(type="output"))) - -flag_choices = set() -for m, c in module_loader.preloaded().items(): - flag_choices.update(set(c.get("flags", []))) - - -class BBOTArgumentParser(argparse.ArgumentParser): - _dummy = False - - def parse_args(self, *args, **kwargs): - """ - Allow space or comma-separated entries for modules and targets - For targets, also allow input files containing additional targets - """ - ret = super().parse_args(*args, **kwargs) - # silent implies -y - if ret.silent: - ret.yes = True - ret.modules = chain_lists(ret.modules) - ret.exclude_modules = chain_lists(ret.exclude_modules) - ret.output_modules = chain_lists(ret.output_modules) - ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}") - ret.whitelist = chain_lists(ret.whitelist, try_files=True, msg="Reading whitelist from file: {filename}") - ret.blacklist = chain_lists(ret.blacklist, try_files=True, msg="Reading blacklist from file: {filename}") - ret.flags = chain_lists(ret.flags) - ret.exclude_flags = chain_lists(ret.exclude_flags) - ret.require_flags = chain_lists(ret.require_flags) - for m in ret.modules: - if m not in module_choices and not self._dummy: - match_and_exit(m, module_choices, msg="module") - for m in ret.exclude_modules: - if m not in module_choices and not self._dummy: - match_and_exit(m, module_choices, msg="module") - for m in ret.output_modules: - if m not in output_module_choices and not self._dummy: - match_and_exit(m, output_module_choices, msg="output module") - for f in set(ret.flags + ret.require_flags): - if f not in flag_choices and not self._dummy: - if f not in flag_choices and not self._dummy: - match_and_exit(f, flag_choices, msg="flag") - return ret - - -class DummyArgumentParser(BBOTArgumentParser): - _dummy = True - - def error(self, message): - pass - - -scan_examples = [ - ( - "Subdomains", - "Perform a full subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum", - ), - ( - "Subdomains (passive only)", - "Perform a passive-only subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum -rf passive", - ), - ( - "Subdomains + port scan + web screenshots", - "Port-scan every subdomain, screenshot every webpage, output to current directory", - "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .", - ), - ( - "Subdomains + basic web scan", - "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules", - "bbot -t evilcorp.com -f subdomain-enum web-basic", - ), - ( - "Web spider", - "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.", - "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2", - ), - ( - "Everything everywhere all at once", - "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei", - "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly", - ), -] - -usage_examples = [ - ( - "List modules", - "", - "bbot -l", - ), - ( - "List flags", - "", - "bbot -lf", - ), -] - - -epilog = "EXAMPLES\n" -for example in (scan_examples, usage_examples): - for title, description, command in example: - epilog += f"\n {title}:\n {command}\n" - - -parser = BBOTArgumentParser( - description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog -) -dummy_parser = DummyArgumentParser( - description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog -) -for p in (parser, dummy_parser): - p.add_argument("--help-all", action="store_true", help="Display full help including module config options") - target = p.add_argument_group(title="Target") - target.add_argument("-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET") - target.add_argument( - "-w", - "--whitelist", - nargs="+", - default=[], - help="What's considered in-scope (by default it's the same as --targets)", - ) - target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things") - target.add_argument( - "--strict-scope", - action="store_true", - help="Don't consider subdomains of target/whitelist to be in-scope", - ) - modules = p.add_argument_group(title="Modules") - modules.add_argument( - "-m", - "--modules", - nargs="+", - default=[], - help=f'Modules to enable. Choices: {",".join(module_choices)}', - metavar="MODULE", - ) - modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") - modules.add_argument( - "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" - ) - modules.add_argument( - "-f", - "--flags", - nargs="+", - default=[], - help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}', - metavar="FLAG", - ) - modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") - modules.add_argument( - "-rf", - "--require-flags", - nargs="+", - default=[], - help=f"Only enable modules with these flags (e.g. -rf passive)", - metavar="FLAG", - ) - modules.add_argument( - "-ef", - "--exclude-flags", - nargs="+", - default=[], - help=f"Disable modules with these flags. (e.g. -ef aggressive)", - metavar="FLAG", - ) - modules.add_argument( - "-om", - "--output-modules", - nargs="+", - default=["human", "json", "csv"], - help=f'Output module(s). Choices: {",".join(output_module_choices)}', - metavar="MODULE", - ) - modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") - scan = p.add_argument_group(title="Scan") - scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") - scan.add_argument( - "-o", - "--output-dir", - metavar="DIR", - ) - scan.add_argument( - "-c", - "--config", - nargs="*", - help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", - metavar="CONFIG", - ) - scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") - scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") - scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") - scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") - scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") - scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") - scan.add_argument( - "--current-config", - action="store_true", - help="Show current config in YAML format", - ) - deps = p.add_argument_group( - title="Module dependencies", description="Control how modules install their dependencies" - ) - g2 = deps.add_mutually_exclusive_group() - g2.add_argument("--no-deps", action="store_true", help="Don't install module dependencies") - g2.add_argument("--force-deps", action="store_true", help="Force install all module dependencies") - g2.add_argument("--retry-deps", action="store_true", help="Try again to install failed module dependencies") - g2.add_argument( - "--ignore-failed-deps", action="store_true", help="Run modules even if they have failed dependencies" - ) - g2.add_argument("--install-all-deps", action="store_true", help="Install dependencies for all modules") - agent = p.add_argument_group(title="Agent", description="Report back to a central server") - agent.add_argument("-a", "--agent-mode", action="store_true", help="Start in agent mode") - misc = p.add_argument_group(title="Misc") - misc.add_argument("--version", action="store_true", help="show BBOT version and exit") - - -cli_options = None -with suppress(Exception): - cli_options = dummy_parser.parse_args() - - -cli_config = [] - - -def get_config(): - global cli_config - with suppress(Exception): - if cli_options.config: - cli_config = cli_options.config - if cli_config: - filename = Path(cli_config[0]).resolve() - if len(cli_config) == 1 and is_file(filename): - try: - conf = OmegaConf.load(str(filename)) - log_to_stderr(f"Loaded custom config from {filename}") - return conf - except Exception as e: - log_to_stderr(f"Error parsing custom config at {filename}: {e}", level="ERROR") - sys.exit(2) - try: - return OmegaConf.from_cli(cli_config) - except Exception as e: - log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") - sys.exit(2) diff --git a/bbot/core/configurator/files.py b/bbot/core/configurator/files.py deleted file mode 100644 index e56950597..000000000 --- a/bbot/core/configurator/files.py +++ /dev/null @@ -1,40 +0,0 @@ -import sys -from pathlib import Path -from omegaconf import OmegaConf - -from ..helpers.misc import mkdir -from ..errors import ConfigLoadError -from ..helpers.logger import log_to_stderr - -config_dir = (Path.home() / ".config" / "bbot").resolve() -defaults_filename = (Path(__file__).parent.parent.parent / "defaults.yml").resolve() -mkdir(config_dir) -config_filename = (config_dir / "bbot.yml").resolve() -secrets_filename = (config_dir / "secrets.yml").resolve() -default_config = None - - -def _get_config(filename, name="config"): - notify = False - if sys.argv and sys.argv[0].endswith("bbot") and not any(x in sys.argv for x in ("-s", "--silent")): - notify = True - filename = Path(filename).resolve() - try: - conf = OmegaConf.load(str(filename)) - if notify and __name__ == "__main__": - log_to_stderr(f"Loaded {name} from {filename}") - return conf - except Exception as e: - if filename.exists(): - raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - return OmegaConf.create() - - -def get_config(): - global default_config - default_config = _get_config(defaults_filename, name="defaults") - return OmegaConf.merge( - default_config, - _get_config(config_filename, name="config"), - _get_config(secrets_filename, name="secrets"), - ) diff --git a/bbot/core/core.py b/bbot/core/core.py new file mode 100644 index 000000000..39517b19f --- /dev/null +++ b/bbot/core/core.py @@ -0,0 +1,63 @@ +from pathlib import Path + + +class BBOTCore: + + def __init__(self): + self._args = None + self._files_config = None + self._module_loader = None + + self.bbot_sudo_pass = None + + self._config = None + + # first, we load config files + # - ensure bbot home directory (needed for logging) + # - ensure module load directories (needed for preloading modules) + + ### to save on performance, we stop here + ### the rest of the attributes populate lazily only when accessed + ### we do this to minimize the time it takes to import bbot as a code library + + # next, we preload modules (needed for parsing CLI args) + # self.load_module_configs() + + # next, we load environment variables + # todo: automatically propagate config values to environ? (would require __setitem__ hooks) + # self.load_environ() + + # finally, we parse CLI args + # self.parse_cli_args() + + @property + def config(self): + if self._config is None: + from .config import files + from .config.logger import BBOTLogger + + self._config = files.get_config() + self.logger = BBOTLogger(self) + return self._config + + @property + def module_loader(self): + if self._module_loader is None: + from .modules import ModuleLoader + + # PRESET TODO: custom module load paths + module_dirs = self.config.get("module_dirs", []) + module_dirs = [Path(p) for p in module_dirs] + module_dirs = list(set(module_dirs)) + + self._module_loader = ModuleLoader(module_dirs=module_dirs) + + return self._module_loader + + @property + def args(self): + if self._args is None: + from .config.args import BBOTArgs + + self._args = BBOTArgs(self) + return self._args diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 00662b969..70f18a2dd 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -13,8 +13,8 @@ from ansible_runner.interface import run from subprocess import CalledProcessError -from bbot.core import configurator -from bbot.modules import module_loader +from bbot.core import CORE + from ..misc import can_sudo_without_password, os_platform log = logging.getLogger("bbot.core.helpers.depsinstaller") @@ -32,8 +32,8 @@ def __init__(self, parent_helper): self._installed_sudo_askpass = False self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None) if self._sudo_password is None: - if configurator.bbot_sudo_pass is not None: - self._sudo_password = configurator.bbot_sudo_pass + if CORE.bbot_sudo_pass is not None: + self._sudo_password = CORE.bbot_sudo_pass elif can_sudo_without_password(): self._sudo_password = "" self.data_dir = self.parent_helper.cache_dir / "depsinstaller" @@ -52,7 +52,8 @@ def __init__(self, parent_helper): if sys.prefix != sys.base_prefix: self.venv = sys.prefix - self.all_modules_preloaded = module_loader.preloaded() + # PRESET TODO: revisit this + self.all_modules_preloaded = CORE.module_loader.preloaded() self.ensure_root_lock = Lock() @@ -310,7 +311,7 @@ def ensure_root(self, message=""): if self.parent_helper.verify_sudo_password(password): log.success("Authentication successful") self._sudo_password = password - configurator.bbot_sudo_pass = password + CORE.bbot_sudo_pass = password else: log.warning("Incorrect password") diff --git a/bbot/core/logger/__init__.py b/bbot/core/logger/__init__.py deleted file mode 100644 index 39f447d6a..000000000 --- a/bbot/core/logger/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .logger import ( - init_logging, - get_log_level, - set_log_level, - add_log_handler, - ColoredFormatter, - get_log_handlers, - toggle_log_level, - remove_log_handler, -) diff --git a/bbot/core/logger/logger.py b/bbot/core/logger/logger.py deleted file mode 100644 index eb8da4c55..000000000 --- a/bbot/core/logger/logger.py +++ /dev/null @@ -1,238 +0,0 @@ -import os -import sys -import logging -from copy import copy -import logging.handlers -from pathlib import Path - -from ..configurator import config -from ..helpers.misc import mkdir, error_and_exit -from ..helpers.logger import colorize, loglevel_mapping - - -_log_level_override = None - -bbot_loggers = None -bbot_log_handlers = None - -debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s") - - -class ColoredFormatter(logging.Formatter): - """ - Pretty colors for terminal - """ - - formatter = logging.Formatter("%(levelname)s %(message)s") - module_formatter = logging.Formatter("%(levelname)s %(name)s: %(message)s") - - def format(self, record): - colored_record = copy(record) - levelname = colored_record.levelname - levelshort = loglevel_mapping.get(levelname, "INFO") - colored_record.levelname = colorize(f"[{levelshort}]", level=levelname) - if levelname == "CRITICAL" or levelname.startswith("HUGE"): - colored_record.msg = colorize(colored_record.msg, level=levelname) - # remove name - if colored_record.name.startswith("bbot.modules."): - colored_record.name = colored_record.name.split("bbot.modules.")[-1] - return self.module_formatter.format(colored_record) - return self.formatter.format(colored_record) - - -def addLoggingLevel(levelName, levelNum, methodName=None): - """ - Comprehensively adds a new logging level to the `logging` module and the - currently configured logging class. - - `levelName` becomes an attribute of the `logging` module with the value - `levelNum`. `methodName` becomes a convenience method for both `logging` - itself and the class returned by `logging.getLoggerClass()` (usually just - `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is - used. - - To avoid accidental clobberings of existing attributes, this method will - raise an `AttributeError` if the level name is already an attribute of the - `logging` module or if the method name is already present - - Example - ------- - >>> addLoggingLevel('TRACE', logging.DEBUG - 5) - >>> logging.getLogger(__name__).setLevel('TRACE') - >>> logging.getLogger(__name__).trace('that worked') - >>> logging.trace('so did this') - >>> logging.TRACE - 5 - - """ - if not methodName: - methodName = levelName.lower() - - if hasattr(logging, levelName): - raise AttributeError(f"{levelName} already defined in logging module") - if hasattr(logging, methodName): - raise AttributeError(f"{methodName} already defined in logging module") - if hasattr(logging.getLoggerClass(), methodName): - raise AttributeError(f"{methodName} already defined in logger class") - - # This method was inspired by the answers to Stack Overflow post - # http://stackoverflow.com/q/2183233/2988730, especially - # http://stackoverflow.com/a/13638084/2988730 - def logForLevel(self, message, *args, **kwargs): - if self.isEnabledFor(levelNum): - self._log(levelNum, message, args, **kwargs) - - def logToRoot(message, *args, **kwargs): - logging.log(levelNum, message, *args, **kwargs) - - logging.addLevelName(levelNum, levelName) - setattr(logging, levelName, levelNum) - setattr(logging.getLoggerClass(), methodName, logForLevel) - setattr(logging, methodName, logToRoot) - - -# custom logging levels -addLoggingLevel("STDOUT", 100) -addLoggingLevel("TRACE", 49) -addLoggingLevel("HUGEWARNING", 31) -addLoggingLevel("HUGESUCCESS", 26) -addLoggingLevel("SUCCESS", 25) -addLoggingLevel("HUGEINFO", 21) -addLoggingLevel("HUGEVERBOSE", 16) -addLoggingLevel("VERBOSE", 15) - - -verbosity_levels_toggle = [logging.INFO, logging.VERBOSE, logging.DEBUG] - - -def get_bbot_loggers(): - global bbot_loggers - if bbot_loggers is None: - bbot_loggers = [ - logging.getLogger("bbot"), - logging.getLogger("asyncio"), - ] - return bbot_loggers - - -def add_log_handler(handler, formatter=None): - if handler.formatter is None: - handler.setFormatter(debug_format) - for logger in get_bbot_loggers(): - if handler not in logger.handlers: - logger.addHandler(handler) - - -def remove_log_handler(handler): - for logger in get_bbot_loggers(): - if handler in logger.handlers: - logger.removeHandler(handler) - - -def init_logging(): - # Don't do this more than once - if len(logging.getLogger("bbot").handlers) == 0: - for logger in get_bbot_loggers(): - include_logger(logger) - - -def include_logger(logger): - bbot_loggers = get_bbot_loggers() - if logger not in bbot_loggers: - bbot_loggers.append(logger) - logger.setLevel(get_log_level()) - for handler in get_log_handlers().values(): - logger.addHandler(handler) - - -def get_log_handlers(): - global bbot_log_handlers - - if bbot_log_handlers is None: - log_dir = Path(config["home"]) / "logs" - if not mkdir(log_dir, raise_error=False): - error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})") - - # Main log file - main_handler = logging.handlers.TimedRotatingFileHandler( - f"{log_dir}/bbot.log", when="d", interval=1, backupCount=14 - ) - - # Separate log file for debugging - debug_handler = logging.handlers.TimedRotatingFileHandler( - f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14 - ) - - def stderr_filter(record): - log_level = get_log_level() - if record.levelno == logging.STDOUT or (record.levelno == logging.TRACE and log_level > logging.DEBUG): - return False - if record.levelno < log_level: - return False - return True - - # Log to stderr - stderr_handler = logging.StreamHandler(sys.stderr) - stderr_handler.addFilter(stderr_filter) - # Log to stdout - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.addFilter(lambda x: x.levelno == logging.STDOUT) - # log to files - debug_handler.addFilter( - lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE and x.levelno != logging.STDOUT) - ) - main_handler.addFilter( - lambda x: x.levelno not in (logging.STDOUT, logging.TRACE) and x.levelno >= logging.VERBOSE - ) - - # Set log format - debug_handler.setFormatter(debug_format) - main_handler.setFormatter(debug_format) - stderr_handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s: %(message)s")) - stdout_handler.setFormatter(logging.Formatter("%(message)s")) - - bbot_log_handlers = { - "stderr": stderr_handler, - "stdout": stdout_handler, - "file_debug": debug_handler, - "file_main": main_handler, - } - return bbot_log_handlers - - -def get_log_level(): - if _log_level_override is not None: - return _log_level_override - - from bbot.core.configurator.args import cli_options - - if config.get("debug", False) or os.environ.get("BBOT_DEBUG", "").lower() in ("true", "yes"): - return logging.DEBUG - - loglevel = logging.INFO - if cli_options is not None: - if cli_options.verbose: - loglevel = logging.VERBOSE - if cli_options.debug: - loglevel = logging.DEBUG - return loglevel - - -def set_log_level(level, logger=None): - global _log_level_override - if logger is not None: - logger.hugeinfo(f"Setting log level to {logging.getLevelName(level)}") - config["silent"] = False - _log_level_override = level - for logger in bbot_loggers: - logger.setLevel(level) - - -def toggle_log_level(logger=None): - log_level = get_log_level() - if log_level in verbosity_levels_toggle: - for i, level in enumerate(verbosity_levels_toggle): - if log_level == level: - set_log_level(verbosity_levels_toggle[(i + 1) % len(verbosity_levels_toggle)], logger=logger) - else: - set_log_level(verbosity_levels_toggle[0], logger=logger) diff --git a/bbot/core/helpers/modules.py b/bbot/core/modules.py similarity index 87% rename from bbot/core/helpers/modules.py rename to bbot/core/modules.py index c6cc52f42..6773a7647 100644 --- a/bbot/core/helpers/modules.py +++ b/bbot/core/modules.py @@ -1,3 +1,4 @@ +import re import ast import sys import importlib @@ -6,8 +7,12 @@ from omegaconf import OmegaConf from contextlib import suppress -from ..flags import flag_descriptions -from .misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform +from .flags import flag_descriptions +from .helpers.logger import log_to_stderr +from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform + + +bbot_code_dir = Path(__file__).parent.parent class ModuleLoader: @@ -19,23 +24,34 @@ class ModuleLoader: This ensures that all requisite libraries and components are available for the module to function correctly. """ - def __init__(self): - self._preloaded = {} + default_module_dir = bbot_code_dir / "modules" + module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") + + def __init__(self, module_dirs=None): + self.__preloaded = {} self._preloaded_orig = None self._modules = {} self._configs = {} + # start with default module dir + self.module_dirs = {self.default_module_dir} + # include custom ones if requested + if module_dirs: + self.module_dirs.update(set(Path(p) for p in module_dirs)) + # expand to include all recursive dirs + self.module_dirs.update(self.get_recursive_dirs(*self.module_dirs)) + def file_filter(self, file): file = file.resolve() if "templates" in file.parts: return False return file.suffix.lower() == ".py" and file.stem not in ["base", "__init__"] - def preload(self, module_dir): - """Preloads all modules within a directory. + def preload(self): + """Preloads all BBOT modules. - This function recursively iterates through each file in the specified directory - and preloads the BBOT module to gather its meta-information and dependencies. + This function recursively iterates through each file in the module directories + and preloads each BBOT module to gather its meta-information and dependencies. Args: module_dir (str or Path): Directory containing BBOT modules to be preloaded. @@ -52,30 +68,44 @@ def preload(self, module_dir): ... } """ - module_dir = Path(module_dir) - for module_file in list_files(module_dir, filter=self.file_filter): - if module_dir.name == "modules": - namespace = f"bbot.modules" - else: - namespace = f"bbot.modules.{module_dir.name}" - try: - preloaded = self.preload_module(module_file) - module_type = "scan" - if module_dir.name in ("output", "internal"): - module_type = str(module_dir.name) - elif module_dir.name not in ("modules"): - preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) - preloaded["type"] = module_type - preloaded["namespace"] = namespace - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_file.stem] = config - self._preloaded[module_file.stem] = preloaded - except Exception: - print(f"[CRIT] Error preloading {module_file}\n\n{traceback.format_exc()}") - print(f"[CRIT] Error in {module_file.name}") - sys.exit(1) - - return self._preloaded + for module_dir in self.module_dirs: + log_to_stderr(f"Preloading modules from {module_dir}", level="HUGESUCCESS") + for module_file in list_files(module_dir, filter=self.file_filter): + log_to_stderr(f"Preloading module from {module_file}", level="HUGESUCCESS") + try: + preloaded = self.preload_module(module_file) + module_type = "scan" + if module_dir.name in ("output", "internal"): + module_type = str(module_dir.name) + elif module_dir.name not in ("modules"): + preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) + preloaded["type"] = module_type + preloaded["namespace"] = "unknown" + config = OmegaConf.create(preloaded.get("config", {})) + self._configs[module_file.stem] = config + self.__preloaded[module_file.stem] = preloaded + except Exception: + log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") + log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") + sys.exit(1) + + return self.__preloaded + + @property + def _preloaded(self): + if not self.__preloaded: + self.preload() + return self.__preloaded + + def get_recursive_dirs(self, *dirs): + dirs = set(Path(d) for d in dirs) + for d in list(dirs): + if not d.is_dir(): + continue + for p in d.iterdir(): + if p.is_dir() and self.module_dir_regex.match(p.name): + dirs.update(self.get_recursive_dirs(p)) + return dirs def preloaded(self, type=None): preloaded = {} @@ -96,7 +126,7 @@ def configs(self, type=None): def find_and_replace(self, **kwargs): if self._preloaded_orig is None: self._preloaded_orig = dict(self._preloaded) - self._preloaded = search_format_dict(self._preloaded_orig, **kwargs) + self.__preloaded = search_format_dict(self._preloaded_orig, **kwargs) def check_type(self, module, type): return self._preloaded[module]["type"] == type @@ -495,6 +525,3 @@ def filter_modules(self, modules=None, mod_type=None): module_list.sort(key=lambda x: "passive" in x[-1]["flags"]) module_list.sort(key=lambda x: x[-1]["type"], reverse=True) return module_list - - -module_loader = ModuleLoader() diff --git a/bbot/core/preset/args.py b/bbot/core/preset/args.py deleted file mode 100644 index c03647c5f..000000000 --- a/bbot/core/preset/args.py +++ /dev/null @@ -1,256 +0,0 @@ -import sys -import argparse -from pathlib import Path -from omegaconf import OmegaConf -from contextlib import suppress - -from ..helpers.logger import log_to_stderr -from ..helpers.misc import chain_lists, match_and_exit, is_file - - -class BBOTArgumentParser(argparse.ArgumentParser): - _dummy = False - - def __init__(self, *args, **kwargs): - if not self._dummy: - module_loader = kwargs.pop("_module_loader") - self._module_choices = sorted(set(module_loader.configs(type="scan"))) - self._output_module_choices = sorted(set(module_loader.configs(type="output"))) - self._flag_choices = set() - for m, c in module_loader.preloaded().items(): - self._flag_choices.update(set(c.get("flags", []))) - - def parse_args(self, *args, **kwargs): - """ - Allow space or comma-separated entries for modules and targets - For targets, also allow input files containing additional targets - """ - ret = super().parse_args(*args, **kwargs) - # silent implies -y - if ret.silent: - ret.yes = True - ret.modules = chain_lists(ret.modules) - ret.exclude_modules = chain_lists(ret.exclude_modules) - ret.output_modules = chain_lists(ret.output_modules) - ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}") - ret.whitelist = chain_lists(ret.whitelist, try_files=True, msg="Reading whitelist from file: {filename}") - ret.blacklist = chain_lists(ret.blacklist, try_files=True, msg="Reading blacklist from file: {filename}") - ret.flags = chain_lists(ret.flags) - ret.exclude_flags = chain_lists(ret.exclude_flags) - ret.require_flags = chain_lists(ret.require_flags) - if not self._dummy: - for m in ret.modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in ret.exclude_modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in ret.output_modules: - if m not in self._output_module_choices: - match_and_exit(m, self._output_module_choices, msg="output module") - for f in set(ret.flags + ret.require_flags): - if f not in self._flag_choices: - match_and_exit(f, self._flag_choices, msg="flag") - return ret - - -class DummyArgumentParser(BBOTArgumentParser): - _dummy = True - - def error(self, message): - pass - - -scan_examples = [ - ( - "Subdomains", - "Perform a full subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum", - ), - ( - "Subdomains (passive only)", - "Perform a passive-only subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum -rf passive", - ), - ( - "Subdomains + port scan + web screenshots", - "Port-scan every subdomain, screenshot every webpage, output to current directory", - "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .", - ), - ( - "Subdomains + basic web scan", - "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules", - "bbot -t evilcorp.com -f subdomain-enum web-basic", - ), - ( - "Web spider", - "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.", - "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2", - ), - ( - "Everything everywhere all at once", - "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei", - "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly", - ), -] - -usage_examples = [ - ( - "List modules", - "", - "bbot -l", - ), - ( - "List flags", - "", - "bbot -lf", - ), -] - - -epilog = "EXAMPLES\n" -for example in (scan_examples, usage_examples): - for title, description, command in example: - epilog += f"\n {title}:\n {command}\n" - - -parser = BBOTArgumentParser( - description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog -) -dummy_parser = DummyArgumentParser( - description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog -) -for p in (parser, dummy_parser): - p.add_argument("--help-all", action="store_true", help="Display full help including module config options") - target = p.add_argument_group(title="Target") - target.add_argument("-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET") - target.add_argument( - "-w", - "--whitelist", - nargs="+", - default=[], - help="What's considered in-scope (by default it's the same as --targets)", - ) - target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things") - target.add_argument( - "--strict-scope", - action="store_true", - help="Don't consider subdomains of target/whitelist to be in-scope", - ) - modules = p.add_argument_group(title="Modules") - modules.add_argument( - "-m", - "--modules", - nargs="+", - default=[], - help=f'Modules to enable. Choices: {",".join(module_choices)}', - metavar="MODULE", - ) - modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") - modules.add_argument( - "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" - ) - modules.add_argument( - "-f", - "--flags", - nargs="+", - default=[], - help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}', - metavar="FLAG", - ) - modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") - modules.add_argument( - "-rf", - "--require-flags", - nargs="+", - default=[], - help=f"Only enable modules with these flags (e.g. -rf passive)", - metavar="FLAG", - ) - modules.add_argument( - "-ef", - "--exclude-flags", - nargs="+", - default=[], - help=f"Disable modules with these flags. (e.g. -ef aggressive)", - metavar="FLAG", - ) - modules.add_argument( - "-om", - "--output-modules", - nargs="+", - default=["human", "json", "csv"], - help=f'Output module(s). Choices: {",".join(output_module_choices)}', - metavar="MODULE", - ) - modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") - scan = p.add_argument_group(title="Scan") - scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") - scan.add_argument( - "-o", - "--output-dir", - metavar="DIR", - ) - scan.add_argument( - "-c", - "--config", - nargs="*", - help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", - metavar="CONFIG", - ) - scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") - scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") - scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") - scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") - scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") - scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") - scan.add_argument( - "--current-config", - action="store_true", - help="Show current config in YAML format", - ) - deps = p.add_argument_group( - title="Module dependencies", description="Control how modules install their dependencies" - ) - g2 = deps.add_mutually_exclusive_group() - g2.add_argument("--no-deps", action="store_true", help="Don't install module dependencies") - g2.add_argument("--force-deps", action="store_true", help="Force install all module dependencies") - g2.add_argument("--retry-deps", action="store_true", help="Try again to install failed module dependencies") - g2.add_argument( - "--ignore-failed-deps", action="store_true", help="Run modules even if they have failed dependencies" - ) - g2.add_argument("--install-all-deps", action="store_true", help="Install dependencies for all modules") - agent = p.add_argument_group(title="Agent", description="Report back to a central server") - agent.add_argument("-a", "--agent-mode", action="store_true", help="Start in agent mode") - misc = p.add_argument_group(title="Misc") - misc.add_argument("--version", action="store_true", help="show BBOT version and exit") - - -cli_options = None -with suppress(Exception): - cli_options = dummy_parser.parse_args() - - -cli_config = [] - - -def get_config(): - global cli_config - with suppress(Exception): - if cli_options.config: - cli_config = cli_options.config - if cli_config: - filename = Path(cli_config[0]).resolve() - if len(cli_config) == 1 and is_file(filename): - try: - conf = OmegaConf.load(str(filename)) - log_to_stderr(f"Loaded custom config from {filename}") - return conf - except Exception as e: - log_to_stderr(f"Error parsing custom config at {filename}: {e}", level="ERROR") - sys.exit(2) - try: - return OmegaConf.from_cli(cli_config) - except Exception as e: - log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") - sys.exit(2) diff --git a/bbot/core/preset/preset.py b/bbot/core/preset/preset.py deleted file mode 100644 index 1a9c0f3d1..000000000 --- a/bbot/core/preset/preset.py +++ /dev/null @@ -1,34 +0,0 @@ -import omegaconf -from pathlib import Path -from omegaconf import OmegaConf - -from . import files -from ..helpers.misc import mkdir - - -class Preset(omegaconf.dictconfig.DictConfig): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # first, we load config files - # this populates defaults and necessary stuff like: - # - bbot home directory - # - module load directories (needed for preloading modules) - self.load_config_files() - - # next, we load environment variables - # todo: automatically propagate config values to environ? (would require __setitem__ hooks) - # self.load_environ() - - # next, we load module defaults - # this populates valid modules + flags (needed for parsing CLI args) - # self.load_module_configs() - - # finally, we parse CLI args - # self.parse_cli_args() - - def load_config_files(self): - self.update(files.get_config()) - - def load_cli_args(self): - pass diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 3ac60741b..1c0f69205 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -33,6 +33,8 @@ web_spider_links_per_page: 25 ### ADVANCED OPTIONS ### +module_paths: [] + # How far out from the main scope to search scope_search_distance: 0 # How far out from the main scope to resolve DNS names / IPs diff --git a/bbot/modules/__init__.py b/bbot/modules/__init__.py index 6062b0170..e69de29bb 100644 --- a/bbot/modules/__init__.py +++ b/bbot/modules/__init__.py @@ -1,14 +0,0 @@ -import re -from pathlib import Path -from bbot.core.helpers.modules import module_loader - -dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") - -parent_dir = Path(__file__).parent.resolve() -module_dirs = set([parent_dir]) -for e in parent_dir.iterdir(): - if e.is_dir() and dir_regex.match(e.name) and not e.name == "modules": - module_dirs.add(e) - -for d in module_dirs: - module_loader.preload(d) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 31bc37680..aa77a28ba 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -12,33 +12,21 @@ from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor -from bbot import config as bbot_config +from bbot.core import CORE from .target import Target from .stats import ScanStats from .manager import ScanManager from .dispatcher import Dispatcher -from bbot.modules import module_loader from bbot.core.event import make_event from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.helper import ConfigAwareHelper from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen -from bbot.core.configurator.environ import prepare_environment from bbot.core.errors import BBOTError, ScanError, ValidationError -from bbot.core.logger import ( - init_logging, - get_log_level, - set_log_level, - add_log_handler, - get_log_handlers, - remove_log_handler, -) log = logging.getLogger("bbot.scanner") -init_logging() - class Scanner: """A class representing a single BBOT scan @@ -158,10 +146,12 @@ def __init__( config = OmegaConf.create({}) else: config = OmegaConf.create(config) - self.config = OmegaConf.merge(bbot_config, config) - prepare_environment(self.config) + self.config = OmegaConf.merge(CORE.config, config) + + # PRESET TODO: revisit this + CORE.prepare_environment(self.config) if self.config.get("debug", False): - set_log_level(logging.DEBUG) + CORE.logger.set_log_level(logging.DEBUG) self.strict_scope = strict_scope self.force_start = force_start @@ -946,7 +936,7 @@ def log_level(self): """ Return the current log level, e.g. logging.INFO """ - return get_log_level() + return CORE.log_level @property def _log_handlers(self): @@ -966,26 +956,27 @@ def _log_handlers(self): return self.__log_handlers def _start_log_handlers(self): + # PRESET TODO: revisit scan logging # add log handlers for handler in self._log_handlers: - add_log_handler(handler) + CORE.logger.add_log_handler(handler) # temporarily disable main ones for handler_name in ("file_main", "file_debug"): - handler = get_log_handlers().get(handler_name, None) + handler = CORE.logger.log_handlers.get(handler_name, None) if handler is not None and handler not in self._log_handler_backup: self._log_handler_backup.append(handler) - remove_log_handler(handler) + CORE.logger.remove_log_handler(handler) def _stop_log_handlers(self): # remove log handlers for handler in self._log_handlers: - remove_log_handler(handler) + CORE.logger.remove_log_handler(handler) # restore main ones for handler in self._log_handler_backup: - add_log_handler(handler) + CORE.logger.add_log_handler(handler) def _internal_modules(self): - for modname in module_loader.preloaded(type="internal"): + for modname in CORE.module_loader.preloaded(type="internal"): if self.config.get(modname, True): yield modname @@ -1008,7 +999,7 @@ def _load_modules(self, modules): modules = [str(m) for m in modules] loaded_modules = {} failed = set() - for module_name, module_class in module_loader.load_modules(modules).items(): + for module_name, module_class in CORE.module_loader.load_modules(modules).items(): if module_class: try: loaded_modules[module_name] = module_class(self) diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index dcf9cd710..2c2214e7e 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -5,8 +5,8 @@ import yaml from pathlib import Path -from bbot.modules import module_loader -from bbot.core.configurator.args import parser, scan_examples +# PRESET TODO: revisit this +from bbot.core import CORE os.environ["BBOT_TABLE_FORMAT"] = "github" @@ -49,6 +49,7 @@ def find_replace_file(file, keyword, replace): f.write(new_content) +# PRESET TODO: revisit this def update_docs(): md_files = [p for p in bbot_code_dir.glob("**/*.md") if p.is_file()] @@ -63,12 +64,12 @@ def update_individual_module_options(): content = f.read() for match in regex.finditer(content): module_name = match.groups()[0].lower() - bbot_module_options_table = module_loader.modules_options_table(modules=[module_name]) + bbot_module_options_table = CORE.module_loader.modules_options_table(modules=[module_name]) find_replace_file(file, f"BBOT MODULE OPTIONS {module_name.upper()}", bbot_module_options_table) # Example commands bbot_example_commands = [] - for title, description, command in scan_examples: + for title, description, command in CORE.args.scan_examples: example = "" example += f"**{title}:**\n\n" # example += f"{description}\n" @@ -79,29 +80,29 @@ def update_individual_module_options(): update_md_files("BBOT EXAMPLE COMMANDS", bbot_example_commands) # Help output - bbot_help_output = parser.format_help().replace("docs.py", "bbot") + bbot_help_output = CORE.args.parser.format_help().replace("docs.py", "bbot") bbot_help_output = f"```text\n{bbot_help_output}\n```" assert len(bbot_help_output.splitlines()) > 50 update_md_files("BBOT HELP OUTPUT", bbot_help_output) # BBOT events - bbot_event_table = module_loader.events_table() + bbot_event_table = CORE.module_loader.events_table() assert len(bbot_event_table.splitlines()) > 10 update_md_files("BBOT EVENTS", bbot_event_table) # BBOT modules - bbot_module_table = module_loader.modules_table() + bbot_module_table = CORE.module_loader.modules_table() assert len(bbot_module_table.splitlines()) > 50 update_md_files("BBOT MODULES", bbot_module_table) # BBOT module options - bbot_module_options_table = module_loader.modules_options_table() + bbot_module_options_table = CORE.module_loader.modules_options_table() assert len(bbot_module_options_table.splitlines()) > 100 update_md_files("BBOT MODULE OPTIONS", bbot_module_options_table) update_individual_module_options() # BBOT module flags - bbot_module_flags_table = module_loader.flags_table() + bbot_module_flags_table = CORE.module_loader.flags_table() assert len(bbot_module_flags_table.splitlines()) > 10 update_md_files("BBOT MODULE FLAGS", bbot_module_flags_table) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index a08ef3ece..a3085d7e4 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -27,7 +27,6 @@ def match_data(self, request: Request) -> bool: os.environ["BBOT_DEBUG"] = "True" from .bbot_fixtures import * # noqa: F401 -import bbot.core.logger # noqa: F401 from bbot.core.errors import * # noqa: F401 # silence pytest_httpserver @@ -225,8 +224,10 @@ def agent(monkeypatch, bbot_config): # bbot config -from bbot import config as default_config +from bbot.core import CORE +# PRESET TODO: revisit this +default_config = CORE.config test_config = OmegaConf.load(Path(__file__).parent / "test.conf") test_config = OmegaConf.merge(default_config, test_config) @@ -239,11 +240,10 @@ def bbot_config(): return test_config -from bbot.modules import module_loader - -available_modules = list(module_loader.configs(type="scan")) -available_output_modules = list(module_loader.configs(type="output")) -available_internal_modules = list(module_loader.configs(type="internal")) +# PRESET TODO: revisit this +available_modules = list(CORE.module_loader.configs(type="scan")) +available_output_modules = list(CORE.module_loader.configs(type="output")) +available_internal_modules = list(CORE.module_loader.configs(type="internal")) @pytest.fixture(autouse=True) diff --git a/bbot/test/test_step_1/test__module__tests.py b/bbot/test/test_step_1/test__module__tests.py index 0d0855557..8a225ef68 100644 --- a/bbot/test/test_step_1/test__module__tests.py +++ b/bbot/test/test_step_1/test__module__tests.py @@ -2,8 +2,8 @@ import importlib from pathlib import Path +from bbot.core import CORE from ..test_step_2.module_tests.base import ModuleTestBase -from bbot.modules import module_loader log = logging.getLogger("bbot.test.modules") @@ -16,7 +16,7 @@ def test__module__tests(): # make sure each module has a .py file - for module_name in module_loader.preloaded(): + for module_name in CORE.module_loader.preloaded(): module_name = module_name.lower() assert module_name in module_test_files, f'No test file found for module "{module_name}"' diff --git a/bbot/test/test_step_1/test_docs.py b/bbot/test/test_step_1/test_docs.py index 6b00d2d0d..a86947ff0 100644 --- a/bbot/test/test_step_1/test_docs.py +++ b/bbot/test/test_step_1/test_docs.py @@ -1,5 +1,4 @@ -from bbot.scripts.docs import update_docs - - def test_docs(): + from bbot.scripts.docs import update_docs + update_docs() diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py new file mode 100644 index 000000000..53971eecd --- /dev/null +++ b/bbot/test/test_step_1/test_presets.py @@ -0,0 +1,21 @@ +def test_presets(): + from bbot.core import CORE + + assert "module_paths" in CORE.config + assert CORE.module_loader.module_paths + assert any(str(x).endswith("/modules") for x in CORE.module_loader.module_paths) + assert "HTTP_RESPONSE" in CORE.config.omit_event_types + + # make sure .copy() works as intended + # preset_copy = CORE.copy() + # assert isinstance(preset_copy, CORE.__class__) + # base_tests(CORE) + # base_tests(preset_copy) + # preset_copy.update({"asdf": {"fdsa": "1234"}}) + # assert "asdf" in preset_copy + # assert preset_copy.asdf.fdsa == "1234" + # assert not "asdf" in CORE + + # preset_copy["testing"] = {"test1": "value"} + # assert "testing" in preset_copy + # assert "testing" not in CORE diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index a4562cfc7..8e7701378 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from bbot.scanner import Scanner -from bbot.modules import module_loader +from bbot.core import CORE from bbot.core.helpers.misc import rand_string from ...bbot_fixtures import test_config @@ -58,7 +58,8 @@ def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, mon self.httpserver_ssl = httpserver_ssl self.monkeypatch = monkeypatch self.request_fixture = request - self.preloaded = module_loader.preloaded() + # PRESET TODO: revisit this + self.preloaded = CORE.module_loader.preloaded() # handle output, internal module types output_modules = None From 6a02e2c7c17633659990cc1741a4903170b15769 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 9 Feb 2024 17:14:04 -0500 Subject: [PATCH 007/171] semi-working preset checkpoint --- bbot/cli.py | 363 +++++++++++++++++++++++++++++++++++- bbot/core/config/args.py | 132 +++++++------ bbot/core/config/environ.py | 254 ++++++++++++------------- bbot/core/config/files.py | 103 +++++++--- bbot/core/core.py | 41 +++- bbot/core/modules.py | 8 +- bbot/scanner/scanner.py | 2 +- 7 files changed, 684 insertions(+), 219 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index bf95a67cc..477c6a98b 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,27 +1,376 @@ #!/usr/bin/env python3 +import os +import re import sys import asyncio import logging import traceback -from bbot.core.helpers.logger import log_to_stderr +from omegaconf import OmegaConf +from contextlib import suppress +from aioconsole import stream # fix tee buffering sys.stdout.reconfigure(line_buffering=True) -log = logging.getLogger("bbot.cli") -sys.stdout.reconfigure(line_buffering=True) +from bbot.core import CORE +from bbot.core import errors +from bbot import __version__ +from bbot.core.helpers.misc import smart_decode +from bbot.core.helpers.logger import log_to_stderr -from bbot.core import CORE +log = logging.getLogger("bbot.cli") + +err = False +scan_name = "" async def _main(): - CORE.args - CORE.module_loader.preloaded() + global err + global scan_name + + CORE.cli_execution = True + + # async def monitor_tasks(): + # in_row = 0 + # while 1: + # try: + # print('looooping') + # tasks = asyncio.all_tasks() + # current_task = asyncio.current_task() + # if len(tasks) == 1 and list(tasks)[0] == current_task: + # print('no tasks') + # in_row += 1 + # else: + # in_row = 0 + # for t in tasks: + # print(t) + # if in_row > 2: + # break + # await asyncio.sleep(1) + # except BaseException as e: + # print(traceback.format_exc()) + # with suppress(BaseException): + # await asyncio.sleep(.1) + + # monitor_tasks_task = asyncio.create_task(monitor_tasks()) + + try: + if len(sys.argv) == 1: + CORE.args.parser.print_help() + sys.exit(1) + + # command-line options are in bbot/core/config/args.py + CORE.args.validate() + options = CORE.args.parsed + + # --version + if options.version: + log.stdout(__version__) + sys.exit(0) + return + + # --current-config + if options.current_config: + log.stdout(f"{OmegaConf.to_yaml(CORE.config)}") + sys.exit(0) + return + + if options.agent_mode: + from bbot.agent import Agent + + agent = Agent(CORE.config) + success = agent.setup() + if success: + await agent.start() + + else: + from bbot.scanner import Scanner + + try: + output_modules = set(options.output_modules) + module_filtering = False + if (options.list_modules or options.help_all) and not any([options.flags, options.modules]): + module_filtering = True + modules = set(CORE.module_loader.preloaded(type="scan")) + else: + modules = set(options.modules) + # enable modules by flags + for m, c in CORE.module_loader.preloaded().items(): + module_type = c.get("type", "scan") + if m not in modules: + flags = c.get("flags", []) + if "deadly" in flags: + continue + for f in options.flags: + if f in flags: + log.verbose(f'Enabling {m} because it has flag "{f}"') + if module_type == "output": + output_modules.add(m) + else: + modules.add(m) + + default_output_modules = ["human", "json", "csv"] + + # Make a list of the modules which can be output to the console + consoleable_output_modules = [ + k for k, v in CORE.module_loader.preloaded(type="output").items() if "console" in v["config"] + ] + + # if none of the output modules provided on the command line are consoleable, don't turn off the defaults. Instead, just add the one specified to the defaults. + if not any(o in consoleable_output_modules for o in output_modules): + output_modules.update(default_output_modules) + + scanner = Scanner( + *options.targets, + modules=list(modules), + output_modules=list(output_modules), + output_dir=options.output_dir, + config=CORE.config, + name=options.name, + whitelist=options.whitelist, + blacklist=options.blacklist, + strict_scope=options.strict_scope, + force_start=options.force, + ) + + if options.install_all_deps: + all_modules = list(CORE.module_loader.preloaded()) + scanner.helpers.depsinstaller.force_deps = True + succeeded, failed = await scanner.helpers.depsinstaller.install(*all_modules) + log.info("Finished installing module dependencies") + return False if failed else True + + scan_name = str(scanner.name) + + # enable modules by dependency + # this is only a basic surface-level check + # todo: recursive dependency graph with networkx or topological sort? + all_modules = list(set(scanner._scan_modules + scanner._internal_modules + scanner._output_modules)) + while 1: + changed = False + dep_choices = CORE.module_loader.recommend_dependencies(all_modules) + if not dep_choices: + break + for event_type, deps in dep_choices.items(): + if event_type in ("*", "all"): + continue + # skip resolving dependency if a target provides the missing type + if any(e.type == event_type for e in scanner.target.events): + continue + required_by = deps.get("required_by", []) + recommended = deps.get("recommended", []) + if not recommended: + log.hugewarning( + f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no modules produce it" + ) + elif len(recommended) == 1: + log.verbose( + f"Enabling {next(iter(recommended))} because {len(required_by):,} modules ({','.join(required_by)}) rely on it for {event_type}" + ) + all_modules = list(set(all_modules + list(recommended))) + scanner._scan_modules = list(set(scanner._scan_modules + list(recommended))) + changed = True + else: + log.hugewarning( + f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no enabled module produces it" + ) + log.hugewarning( + f"Recommend enabling one or more of the following modules which produce {event_type}:" + ) + for m in recommended: + log.warning(f" - {m}") + if not changed: + break + + # required flags + modules = set(scanner._scan_modules) + for m in scanner._scan_modules: + flags = CORE.module_loader._preloaded.get(m, {}).get("flags", []) + if not all(f in flags for f in options.require_flags): + log.verbose( + f"Removing {m} because it does not have the required flags: {'+'.join(options.require_flags)}" + ) + with suppress(KeyError): + modules.remove(m) + + # excluded flags + for m in scanner._scan_modules: + flags = CORE.module_loader._preloaded.get(m, {}).get("flags", []) + if any(f in flags for f in options.exclude_flags): + log.verbose(f"Removing {m} because of excluded flag: {','.join(options.exclude_flags)}") + with suppress(KeyError): + modules.remove(m) + + # excluded modules + for m in options.exclude_modules: + if m in modules: + log.verbose(f"Removing {m} because it is excluded") + with suppress(KeyError): + modules.remove(m) + scanner._scan_modules = list(modules) + + log_fn = log.info + if options.list_modules or options.help_all: + log_fn = log.stdout + + help_modules = list(modules) + if module_filtering: + help_modules = None + + if options.help_all: + log_fn(CORE.args.parser.format_help()) + + if options.list_flags: + log.stdout("") + log.stdout("### FLAGS ###") + log.stdout("") + for row in CORE.module_loader.flags_table(flags=options.flags).splitlines(): + log.stdout(row) + return + + log_fn("") + log_fn("### MODULES ###") + log_fn("") + for row in CORE.module_loader.modules_table(modules=help_modules).splitlines(): + log_fn(row) + + if options.help_all: + log_fn("") + log_fn("### MODULE OPTIONS ###") + log_fn("") + for row in CORE.module_loader.modules_options_table(modules=help_modules).splitlines(): + log_fn(row) + + if options.list_modules or options.list_flags or options.help_all: + return + + module_list = CORE.module_loader.filter_modules(modules=modules) + deadly_modules = [] + active_modules = [] + active_aggressive_modules = [] + slow_modules = [] + for m in module_list: + if m[0] in scanner._scan_modules: + if "deadly" in m[-1]["flags"]: + deadly_modules.append(m[0]) + if "active" in m[-1]["flags"]: + active_modules.append(m[0]) + if "aggressive" in m[-1]["flags"]: + active_aggressive_modules.append(m[0]) + if "slow" in m[-1]["flags"]: + slow_modules.append(m[0]) + if scanner._scan_modules: + if deadly_modules and not options.allow_deadly: + log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") + log.hugewarning(f"Deadly modules are highly intrusive") + log.hugewarning(f"Please specify --allow-deadly to continue") + return False + if active_modules: + if active_modules: + if active_aggressive_modules: + log.hugewarning( + "This is an (aggressive) active scan! Intrusive connections will be made to target" + ) + else: + log.hugewarning( + "This is a (safe) active scan. Non-intrusive connections will be made to target" + ) + else: + log.hugeinfo("This is a passive scan. No connections will be made to target") + if slow_modules: + log.warning( + f"You have enabled the following slow modules: {','.join(slow_modules)}. Scan may take a while" + ) + + scanner.helpers.word_cloud.load() + + await scanner._prep() + + if not options.dry_run: + log.trace(f"Command: {' '.join(sys.argv)}") + + if sys.stdin.isatty(): + if not options.agent_mode and not options.yes: + log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}") + input() + + def handle_keyboard_input(keyboard_input): + kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") + if keyboard_input: + log.verbose(f'Got keyboard input: "{keyboard_input}"') + kill_match = kill_regex.match(keyboard_input) + if kill_match: + module = kill_match.group("module") + if module in scanner.modules: + log.hugewarning(f'Killing module: "{module}"') + scanner.manager.kill_module(module, message="killed by user") + else: + log.warning(f'Invalid module: "{module}"') + else: + CORE.logger.toggle_log_level(logger=log) + scanner.manager.modules_status(_log=True) + + # Reader + reader = stream.StandardStreamReader() + protocol = stream.StandardStreamReaderProtocol(reader) + await asyncio.get_event_loop().connect_read_pipe(lambda: protocol, sys.stdin) + + async def akeyboard_listen(): + try: + allowed_errors = 10 + while 1: + keyboard_input = None + try: + keyboard_input = smart_decode((await reader.readline()).strip()) + allowed_errors = 10 + except Exception as e: + log_to_stderr(f"Error in keyboard listen loop: {e}", level="TRACE") + log_to_stderr(traceback.format_exc(), level="TRACE") + allowed_errors -= 1 + if keyboard_input is not None: + handle_keyboard_input(keyboard_input) + if allowed_errors <= 0: + break + except Exception as e: + log_to_stderr(f"Error in keyboard listen task: {e}", level="ERROR") + log_to_stderr(traceback.format_exc(), level="TRACE") + + asyncio.create_task(akeyboard_listen()) + + await scanner.async_start_without_generator() + + except errors.ScanError as e: + log_to_stderr(str(e), level="ERROR") + except Exception: + raise + + except errors.BBOTError as e: + log_to_stderr(f"{e} (--debug for details)", level="ERROR") + if CORE.logger.log_level <= logging.DEBUG: + log_to_stderr(traceback.format_exc(), level="DEBUG") + err = True + + except Exception: + log_to_stderr(f"Encountered unknown error: {traceback.format_exc()}", level="ERROR") + err = True + + finally: + # save word cloud + with suppress(BaseException): + save_success, filename = scanner.helpers.word_cloud.save() + if save_success: + log_to_stderr(f"Saved word cloud ({len(scanner.helpers.word_cloud):,} words) to {filename}") + # remove output directory if empty + with suppress(BaseException): + scanner.home.rmdir() + if err: + os._exit(1) def main(): + global scan_name try: asyncio.run(_main()) except asyncio.CancelledError: @@ -29,6 +378,8 @@ def main(): log_to_stderr(traceback.format_exc(), level="DEBUG") except KeyboardInterrupt: msg = "Interrupted" + if scan_name: + msg = f"You killed {scan_name}" log_to_stderr(msg, level="WARNING") if CORE.logger.log_level <= logging.DEBUG: log_to_stderr(traceback.format_exc(), level="DEBUG") diff --git a/bbot/core/config/args.py b/bbot/core/config/args.py index e8ed21af5..f36238b87 100644 --- a/bbot/core/config/args.py +++ b/bbot/core/config/args.py @@ -1,8 +1,8 @@ +import re import sys import argparse from pathlib import Path from omegaconf import OmegaConf -from contextlib import suppress from ..helpers.logger import log_to_stderr from ..helpers.misc import chain_lists, match_and_exit, is_file @@ -16,19 +16,8 @@ class BBOTArgumentParser(argparse.ArgumentParser): - option to *not* exit on error """ - def __init__(self, *args, **kwargs): - self._dummy = kwargs.pop("_dummy", False) - self.bbot_core = kwargs.pop("_core") - self._module_choices = sorted(set(self.bbot_core.module_loader.configs(type="scan"))) - self._output_module_choices = sorted(set(self.bbot_core.module_loader.configs(type="output"))) - self._flag_choices = set() - for m, c in self.bbot_core.module_loader.preloaded().items(): - self._flag_choices.update(set(c.get("flags", []))) - super().__init__(*args, **kwargs) - - def error(self, message): - if not self._dummy: - return super().error(message) + # module config options to exclude from validation + exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$") def parse_args(self, *args, **kwargs): """ @@ -48,19 +37,6 @@ def parse_args(self, *args, **kwargs): ret.flags = chain_lists(ret.flags) ret.exclude_flags = chain_lists(ret.exclude_flags) ret.require_flags = chain_lists(ret.require_flags) - if not self._dummy: - for m in ret.modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in ret.exclude_modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in ret.output_modules: - if m not in self._output_module_choices: - match_and_exit(m, self._output_module_choices, msg="output module") - for f in set(ret.flags + ret.require_flags): - if f not in self._flag_choices: - match_and_exit(f, self._flag_choices, msg="flag") return ret @@ -118,43 +94,52 @@ class BBOTArgs: def __init__(self, core): self.core = core - self.parser = self.create_parser(_core=self.core) - self.dummy_parser = self.create_parser(_core=self.core, _dummy=True) self._parsed = None self._cli_config = None + self._module_choices = sorted(set(self.core.module_loader.preloaded(type="scan"))) + self._output_module_choices = sorted(set(self.core.module_loader.preloaded(type="output"))) + self._flag_choices = set() + for m, c in self.core.module_loader.preloaded().items(): + self._flag_choices.update(set(c.get("flags", []))) + self._flag_choices = sorted(self._flag_choices) + + log_to_stderr(f"module options: {self._module_choices}") + log_to_stderr(f"output module options: {self._output_module_choices}") + log_to_stderr(f"flag options: {self._flag_choices}") + + self.parser = self.create_parser() + @property def parsed(self): """ Returns the parsed BBOT Argument Parser. """ if self._parsed is None: - self._parsed = self.dummy_parser.parse_args() + self._parsed = self.parser.parse_args() return self._parsed @property def cli_config(self): - if self._cli_config is None: - with suppress(Exception): - if self.parsed.config: - cli_config = self.parsed.config - if cli_config: - filename = Path(cli_config[0]).resolve() - if len(cli_config) == 1 and is_file(filename): - try: - conf = OmegaConf.load(str(filename)) - log_to_stderr(f"Loaded custom config from {filename}") - return conf - except Exception as e: - log_to_stderr(f"Error parsing custom config at {filename}: {e}", level="ERROR") - sys.exit(2) - try: - self._cli_config = OmegaConf.from_cli(cli_config) - except Exception as e: - log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") - sys.exit(2) if self._cli_config is None: self._cli_config = OmegaConf.create({}) + if self.parsed.config: + for c in self.parsed.config: + config_file = Path(c).resolve() + if config_file.is_file(): + try: + cli_config = OmegaConf.load(str(config_file)) + log_to_stderr(f"Loaded custom config from {config_file}") + except Exception as e: + log_to_stderr(f"Error parsing custom config at {config_file}: {e}", level="ERROR") + sys.exit(2) + else: + try: + cli_config = OmegaConf.from_cli(cli_config) + except Exception as e: + log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") + sys.exit(2) + self._cli_config = OmegaConf.merge(self._cli_config, cli_config) return self._cli_config def create_parser(self, *args, **kwargs): @@ -188,7 +173,7 @@ def create_parser(self, *args, **kwargs): "--modules", nargs="+", default=[], - help=f'Modules to enable. Choices: {",".join(p._module_choices)}', + help=f'Modules to enable. Choices: {",".join(self._module_choices)}', metavar="MODULE", ) modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") @@ -200,7 +185,7 @@ def create_parser(self, *args, **kwargs): "--flags", nargs="+", default=[], - help=f'Enable modules by flag. Choices: {",".join(sorted(p._flag_choices))}', + help=f'Enable modules by flag. Choices: {",".join(self._flag_choices)}', metavar="FLAG", ) modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") @@ -225,7 +210,7 @@ def create_parser(self, *args, **kwargs): "--output-modules", nargs="+", default=["human", "json", "csv"], - help=f'Output module(s). Choices: {",".join(p._output_module_choices)}', + help=f'Output module(s). Choices: {",".join(self._output_module_choices)}', metavar="MODULE", ) modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") @@ -242,6 +227,7 @@ def create_parser(self, *args, **kwargs): nargs="*", help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", metavar="CONFIG", + default=[], ) scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") @@ -270,3 +256,45 @@ def create_parser(self, *args, **kwargs): misc = p.add_argument_group(title="Misc") misc.add_argument("--version", action="store_true", help="show BBOT version and exit") return p + + def validate(self): + """ + Validate command line arguments + - Validate modules/flags to make sure they exist + - If --config was specified, check the config options to make sure they exist + """ + # modules + for m in self.parsed.modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in self.parsed.exclude_modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in self.parsed.output_modules: + if m not in self._output_module_choices: + match_and_exit(m, self._output_module_choices, msg="output module") + # flags + for f in set(self.parsed.flags + self.parsed.require_flags + self.parsed.exclude_flags): + if f not in self._flag_choices: + match_and_exit(f, self._flag_choices, msg="flag") + + # config options + sentinel = object() + conf = [a for a in self.parsed.config if not is_file(a)] + all_options = None + for c in conf: + c = c.split("=")[0].strip() + v = OmegaConf.select(self.core.default_config, c, default=sentinel) + # if option isn't in the default config + if v is sentinel: + if self.exclude_from_validation.match(c): + continue + if all_options is None: + from ...modules import module_loader + + modules_options = set() + for module_options in module_loader.modules_options().values(): + modules_options.update(set(o[0] for o in module_options)) + global_options = set(self.core.default_config.keys()) - {"modules", "output_modules"} + all_options = global_options.union(modules_options) + match_and_exit(c, all_options, msg="module option") diff --git a/bbot/core/config/environ.py b/bbot/core/config/environ.py index 4358bb78d..4d6233232 100644 --- a/bbot/core/config/environ.py +++ b/bbot/core/config/environ.py @@ -3,15 +3,9 @@ import omegaconf from pathlib import Path -from . import args -from ...modules import module_loader from ..helpers.misc import cpu_architecture, os_platform, os_platform_friendly -# keep track of whether BBOT is being executed via the CLI -cli_execution = False - - def increase_limit(new_limit): try: import resource @@ -30,124 +24,130 @@ def increase_limit(new_limit): increase_limit(65535) -def flatten_config(config, base="bbot"): - """ - Flatten a JSON-like config into a list of environment variables: - {"modules": [{"httpx": {"timeout": 5}}]} --> "BBOT_MODULES_HTTPX_TIMEOUT=5" - """ - if type(config) == omegaconf.dictconfig.DictConfig: - for k, v in config.items(): - new_base = f"{base}_{k}" - if type(v) == omegaconf.dictconfig.DictConfig: - yield from flatten_config(v, base=new_base) - elif type(v) != omegaconf.listconfig.ListConfig: - yield (new_base.upper(), str(v)) - - -def add_to_path(v, k="PATH"): - var_list = os.environ.get(k, "").split(":") - deduped_var_list = [] - for _ in var_list: - if not _ in deduped_var_list: - deduped_var_list.append(_) - if not v in deduped_var_list: - deduped_var_list = [v] + deduped_var_list - new_var_str = ":".join(deduped_var_list) - os.environ[k] = new_var_str - - -def prepare_environment(bbot_config): - """ - Sync config to OS environment variables - """ - # ensure bbot_home - if not "home" in bbot_config: - bbot_config["home"] = "~/.bbot" - home = Path(bbot_config["home"]).expanduser().resolve() - bbot_config["home"] = str(home) - - # if we're running in a virtual environment, make sure to include its /bin in PATH - if sys.prefix != sys.base_prefix: - bin_dir = str(Path(sys.prefix) / "bin") - add_to_path(bin_dir) - - # add ~/.local/bin to PATH - local_bin_dir = str(Path.home() / ".local" / "bin") - add_to_path(local_bin_dir) - - # ensure bbot_tools - bbot_tools = home / "tools" - os.environ["BBOT_TOOLS"] = str(bbot_tools) - if not str(bbot_tools) in os.environ.get("PATH", "").split(":"): - os.environ["PATH"] = f'{bbot_tools}:{os.environ.get("PATH", "").strip(":")}' - # ensure bbot_cache - bbot_cache = home / "cache" - os.environ["BBOT_CACHE"] = str(bbot_cache) - # ensure bbot_temp - bbot_temp = home / "temp" - os.environ["BBOT_TEMP"] = str(bbot_temp) - # ensure bbot_lib - bbot_lib = home / "lib" - os.environ["BBOT_LIB"] = str(bbot_lib) - # export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:~/.bbot/lib/ - add_to_path(str(bbot_lib), k="LD_LIBRARY_PATH") - - # platform variables - os.environ["BBOT_OS_PLATFORM"] = os_platform() - os.environ["BBOT_OS"] = os_platform_friendly() - os.environ["BBOT_CPU_ARCH"] = cpu_architecture() - - # exchange certain options between CLI args and config - if cli_execution and args.cli_options is not None: - # deps - bbot_config["retry_deps"] = args.cli_options.retry_deps - bbot_config["force_deps"] = args.cli_options.force_deps - bbot_config["no_deps"] = args.cli_options.no_deps - bbot_config["ignore_failed_deps"] = args.cli_options.ignore_failed_deps - # debug - bbot_config["debug"] = args.cli_options.debug - bbot_config["silent"] = args.cli_options.silent - - import logging - - log = logging.getLogger() - if bbot_config.get("debug", False): - bbot_config["silent"] = False - log = logging.getLogger("bbot") - log.setLevel(logging.DEBUG) - logging.getLogger("asyncio").setLevel(logging.DEBUG) - elif bbot_config.get("silent", False): - log = logging.getLogger("bbot") - log.setLevel(logging.CRITICAL) - - # copy config to environment - bbot_environ = flatten_config(bbot_config) - os.environ.update(bbot_environ) - - # handle HTTP proxy - http_proxy = bbot_config.get("http_proxy", "") - if http_proxy: - os.environ["HTTP_PROXY"] = http_proxy - os.environ["HTTPS_PROXY"] = http_proxy - else: - os.environ.pop("HTTP_PROXY", None) - os.environ.pop("HTTPS_PROXY", None) - - # replace environment variables in preloaded modules - module_loader.find_and_replace(**os.environ) - - # ssl verification - import urllib3 - - urllib3.disable_warnings() - ssl_verify = bbot_config.get("ssl_verify", False) - if not ssl_verify: - import requests - import functools - - requests.adapters.BaseAdapter.send = functools.partialmethod(requests.adapters.BaseAdapter.send, verify=False) - requests.adapters.HTTPAdapter.send = functools.partialmethod(requests.adapters.HTTPAdapter.send, verify=False) - requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) - requests.request = functools.partial(requests.request, verify=False) - - return bbot_config +class BBOTEnviron: + + def __init__(self, core): + self.core = core + + def flatten_config(self, config, base="bbot"): + """ + Flatten a JSON-like config into a list of environment variables: + {"modules": [{"httpx": {"timeout": 5}}]} --> "BBOT_MODULES_HTTPX_TIMEOUT=5" + """ + if type(config) == omegaconf.dictconfig.DictConfig: + for k, v in config.items(): + new_base = f"{base}_{k}" + if type(v) == omegaconf.dictconfig.DictConfig: + yield from self.flatten_config(v, base=new_base) + elif type(v) != omegaconf.listconfig.ListConfig: + yield (new_base.upper(), str(v)) + + def add_to_path(self, v, k="PATH"): + var_list = os.environ.get(k, "").split(":") + deduped_var_list = [] + for _ in var_list: + if not _ in deduped_var_list: + deduped_var_list.append(_) + if not v in deduped_var_list: + deduped_var_list = [v] + deduped_var_list + new_var_str = ":".join(deduped_var_list) + os.environ[k] = new_var_str + + def prepare(self): + """ + Sync config to OS environment variables + """ + # ensure bbot_home + if not "home" in self.core.config: + self.core.config["home"] = "~/.bbot" + home = Path(self.core.config["home"]).expanduser().resolve() + self.core.config["home"] = str(home) + + # if we're running in a virtual environment, make sure to include its /bin in PATH + if sys.prefix != sys.base_prefix: + bin_dir = str(Path(sys.prefix) / "bin") + self.add_to_path(bin_dir) + + # add ~/.local/bin to PATH + local_bin_dir = str(Path.home() / ".local" / "bin") + self.add_to_path(local_bin_dir) + + # ensure bbot_tools + bbot_tools = home / "tools" + os.environ["BBOT_TOOLS"] = str(bbot_tools) + if not str(bbot_tools) in os.environ.get("PATH", "").split(":"): + os.environ["PATH"] = f'{bbot_tools}:{os.environ.get("PATH", "").strip(":")}' + # ensure bbot_cache + bbot_cache = home / "cache" + os.environ["BBOT_CACHE"] = str(bbot_cache) + # ensure bbot_temp + bbot_temp = home / "temp" + os.environ["BBOT_TEMP"] = str(bbot_temp) + # ensure bbot_lib + bbot_lib = home / "lib" + os.environ["BBOT_LIB"] = str(bbot_lib) + # export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:~/.bbot/lib/ + self.add_to_path(str(bbot_lib), k="LD_LIBRARY_PATH") + + # platform variables + os.environ["BBOT_OS_PLATFORM"] = os_platform() + os.environ["BBOT_OS"] = os_platform_friendly() + os.environ["BBOT_CPU_ARCH"] = cpu_architecture() + + # exchange certain options between CLI args and config + # PRESET TODO: do we even need this? + if self.core.cli_execution and self.core.args.cli_config: + # deps + self.core.config["retry_deps"] = self.core.args.cli_options.retry_deps + self.core.config["force_deps"] = self.core.args.cli_options.force_deps + self.core.config["no_deps"] = self.core.args.cli_options.no_deps + self.core.config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps + # debug + self.core.config["debug"] = self.core.args.cli_options.debug + self.core.config["silent"] = self.core.args.cli_options.silent + + import logging + + log = logging.getLogger() + if self.core.config.get("debug", False): + self.core.config["silent"] = False + log = logging.getLogger("bbot") + log.setLevel(logging.DEBUG) + logging.getLogger("asyncio").setLevel(logging.DEBUG) + elif self.core.config.get("silent", False): + log = logging.getLogger("bbot") + log.setLevel(logging.CRITICAL) + + # copy config to environment + bbot_environ = self.flatten_config(self.core.config) + os.environ.update(bbot_environ) + + # handle HTTP proxy + http_proxy = self.core.config.get("http_proxy", "") + if http_proxy: + os.environ["HTTP_PROXY"] = http_proxy + os.environ["HTTPS_PROXY"] = http_proxy + else: + os.environ.pop("HTTP_PROXY", None) + os.environ.pop("HTTPS_PROXY", None) + + # replace environment variables in preloaded modules + self.core.module_loader.find_and_replace(**os.environ) + + # ssl verification + import urllib3 + + urllib3.disable_warnings() + ssl_verify = self.core.config.get("ssl_verify", False) + if not ssl_verify: + import requests + import functools + + requests.adapters.BaseAdapter.send = functools.partialmethod( + requests.adapters.BaseAdapter.send, verify=False + ) + requests.adapters.HTTPAdapter.send = functools.partialmethod( + requests.adapters.HTTPAdapter.send, verify=False + ) + requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) + requests.request = functools.partial(requests.request, verify=False) diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index af1549253..03a227c4c 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -4,32 +4,81 @@ from ..errors import ConfigLoadError from ..helpers.logger import log_to_stderr +from ..helpers.misc import mkdir, clean_dict, filter_dict + bbot_code_dir = Path(__file__).parent.parent.parent -config_dir = (Path.home() / ".config" / "bbot").resolve() -defaults_filename = (bbot_code_dir / "defaults.yml").resolve() -config_filename = (config_dir / "bbot.yml").resolve() -secrets_filename = (config_dir / "secrets.yml").resolve() - - -def _get_config(filename, name="config"): - filename = Path(filename).resolve() - try: - conf = OmegaConf.load(str(filename)) - cli_silent = any(x in sys.argv for x in ("-s", "--silent")) - if __name__ == "__main__" and not cli_silent: - log_to_stderr(f"Loaded {name} from {filename}") - return conf - except Exception as e: - if filename.exists(): - raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - return OmegaConf.create() - - -def get_config(): - default_config = _get_config(defaults_filename, name="defaults") - return OmegaConf.merge( - default_config, - _get_config(config_filename, name="config"), - _get_config(secrets_filename, name="secrets"), - ) + + +class BBOTConfigFiles: + + config_dir = (Path.home() / ".config" / "bbot").resolve() + defaults_filename = (bbot_code_dir / "defaults.yml").resolve() + config_filename = (config_dir / "bbot.yml").resolve() + secrets_filename = (config_dir / "secrets.yml").resolve() + + def __init__(self, core): + self.core = core + + def ensure_config_files(self): + mkdir(self.config_dir) + + secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] + exclude_keys = ["modules", "output_modules", "internal_modules"] + + comment_notice = ( + "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" + + "# Please be sure to uncomment when inserting API keys, etc.\n" + ) + + # ensure bbot.yml + if not self.config_filename.exists(): + log_to_stderr(f"Creating BBOT config at {self.config_filename}") + no_secrets_config = OmegaConf.to_object(self.core.default_config) + no_secrets_config = clean_dict( + no_secrets_config, + *secrets_strings, + fuzzy=True, + exclude_keys=exclude_keys, + ) + yaml = OmegaConf.to_yaml(no_secrets_config) + yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) + with open(str(self.config_filename), "w") as f: + f.write(yaml) + + # ensure secrets.yml + if not self.secrets_filename.exists(): + log_to_stderr(f"Creating BBOT secrets at {self.secrets_filename}") + secrets_only_config = OmegaConf.to_object(self.core.default_config) + secrets_only_config = filter_dict( + secrets_only_config, + *secrets_strings, + fuzzy=True, + exclude_keys=exclude_keys, + ) + yaml = OmegaConf.to_yaml(secrets_only_config) + yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) + with open(str(self.secrets_filename), "w") as f: + f.write(yaml) + self.secrets_filename.chmod(0o600) + + def _get_config(self, filename, name="config"): + filename = Path(filename).resolve() + try: + conf = OmegaConf.load(str(filename)) + cli_silent = any(x in sys.argv for x in ("-s", "--silent")) + if __name__ == "__main__" and not cli_silent: + log_to_stderr(f"Loaded {name} from {filename}") + return conf + except Exception as e: + if filename.exists(): + raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") + return OmegaConf.create() + + def get_config(self): + default_config = self._get_config(self.defaults_filename, name="defaults") + return OmegaConf.merge( + default_config, + self._get_config(self.config_filename, name="config"), + self._get_config(self.secrets_filename, name="secrets"), + ) diff --git a/bbot/core/core.py b/bbot/core/core.py index 39517b19f..a6300cfa4 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -5,12 +5,20 @@ class BBOTCore: def __init__(self): self._args = None + self._logger = None + self._environ = None self._files_config = None self._module_loader = None self.bbot_sudo_pass = None + self.cli_execution = False self._config = None + self._default_config = None + + # PRESET TODO: add back in bbot/core/configurator/__init__.py + # - check_cli_args + # - ensure_config_files # first, we load config files # - ensure bbot home directory (needed for logging) @@ -30,16 +38,35 @@ def __init__(self): # finally, we parse CLI args # self.parse_cli_args() + @property + def files_config(self): + if self._files_config is None: + from .config import files + + self.files = files + self._files_config = files.BBOTConfigFiles(self) + return self._files_config + @property def config(self): if self._config is None: - from .config import files from .config.logger import BBOTLogger - self._config = files.get_config() - self.logger = BBOTLogger(self) + self._config = self.files_config.get_config() + self._default_config = self._config.copy() + self._logger = BBOTLogger(self) return self._config + @property + def default_config(self): + self.config + return self._default_config + + @property + def logger(self): + self.config + return self._logger + @property def module_loader(self): if self._module_loader is None: @@ -54,6 +81,14 @@ def module_loader(self): return self._module_loader + @property + def environ(self): + if self._environ is None: + from .config.environ import BBOTEnviron + + self._environ = BBOTEnviron(self) + return self._environ + @property def args(self): if self._args is None: diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 6773a7647..1b3320535 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -69,9 +69,11 @@ def preload(self): } """ for module_dir in self.module_dirs: - log_to_stderr(f"Preloading modules from {module_dir}", level="HUGESUCCESS") for module_file in list_files(module_dir, filter=self.file_filter): - log_to_stderr(f"Preloading module from {module_file}", level="HUGESUCCESS") + if module_dir.name == "modules": + namespace = f"bbot.modules" + else: + namespace = f"bbot.modules.{module_dir.name}" try: preloaded = self.preload_module(module_file) module_type = "scan" @@ -80,7 +82,7 @@ def preload(self): elif module_dir.name not in ("modules"): preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) preloaded["type"] = module_type - preloaded["namespace"] = "unknown" + preloaded["namespace"] = namespace config = OmegaConf.create(preloaded.get("config", {})) self._configs[module_file.stem] = config self.__preloaded[module_file.stem] = preloaded diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index aa77a28ba..26efba084 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -149,7 +149,7 @@ def __init__( self.config = OmegaConf.merge(CORE.config, config) # PRESET TODO: revisit this - CORE.prepare_environment(self.config) + CORE.environ.prepare() if self.config.get("debug", False): CORE.logger.set_log_level(logging.DEBUG) From d990ccf440b248e0eee448320deb883fc262e6d0 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 12 Feb 2024 13:03:24 -0500 Subject: [PATCH 008/171] adding more possible ajax endpoints --- bbot/modules/ajaxpro.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py index 3a0061315..afdb19b77 100644 --- a/bbot/modules/ajaxpro.py +++ b/bbot/modules/ajaxpro.py @@ -17,22 +17,24 @@ async def handle_event(self, event): if event.type == "URL": if "dir" not in event.tags: return False - probe_url = f"{event.data}ajaxpro/whatever.ashx" - probe = await self.helpers.request(probe_url) - if probe: - if probe.status_code == 200: - probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx") - if probe_confirm: - if probe_confirm.status_code != 200: - await self.emit_event( - { - "host": str(event.host), - "url": event.data, - "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{probe_url}]", - }, - "FINDING", - event, - ) + for stem in ["ajax","ajaxpro"]: + probe_url = f"{event.data}{stem}/whatever.ashx" + self.critical(probe_url) + probe = await self.helpers.request(probe_url) + if probe: + if probe.status_code == 200: + probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx") + if probe_confirm: + if probe_confirm.status_code != 200: + await self.emit_event( + { + "host": str(event.host), + "url": event.data, + "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{probe_url}]", + }, + "FINDING", + event, + ) elif event.type == "HTTP_RESPONSE": resp_body = event.data.get("body", None) From 3a51c1205711125a81087e2b7a38094bf58e3bf5 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 12 Feb 2024 13:03:40 -0500 Subject: [PATCH 009/171] black --- bbot/modules/ajaxpro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py index afdb19b77..4dad1729e 100644 --- a/bbot/modules/ajaxpro.py +++ b/bbot/modules/ajaxpro.py @@ -17,7 +17,7 @@ async def handle_event(self, event): if event.type == "URL": if "dir" not in event.tags: return False - for stem in ["ajax","ajaxpro"]: + for stem in ["ajax", "ajaxpro"]: probe_url = f"{event.data}{stem}/whatever.ashx" self.critical(probe_url) probe = await self.helpers.request(probe_url) From 12423aef3b2026a05a19e88b4c212ad9e0e86372 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 12 Feb 2024 16:59:28 -0500 Subject: [PATCH 010/171] removing debug statement --- bbot/modules/ajaxpro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py index 4dad1729e..46d475cca 100644 --- a/bbot/modules/ajaxpro.py +++ b/bbot/modules/ajaxpro.py @@ -19,7 +19,6 @@ async def handle_event(self, event): return False for stem in ["ajax", "ajaxpro"]: probe_url = f"{event.data}{stem}/whatever.ashx" - self.critical(probe_url) probe = await self.helpers.request(probe_url) if probe: if probe.status_code == 200: From 2787bbaeb68785c59f74f88ac083311ef6d15388 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 13 Feb 2024 10:26:03 -0500 Subject: [PATCH 011/171] don't dedup DNS child events --- bbot/core/helpers/dns.py | 5 +++++ bbot/scanner/manager.py | 1 + 2 files changed, 6 insertions(+) diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 9ad22116f..b81194403 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -334,6 +334,10 @@ async def _resolve_hostname(self, query, **kwargs): if results: self._last_dns_success = time.time() + self.debug(f"Answers for {query} with kwargs={kwargs}: {list(results)}") + + if errors: + self.debug(f"Errors for {query} with kwargs={kwargs}: {errors}") return results, errors @@ -1038,6 +1042,7 @@ def _get_dummy_module(self, name): dummy_module = self._dummy_modules[name] except KeyError: dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") + dummy_module.suppress_dupes = False self._dummy_modules[name] = dummy_module return dummy_module diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index c0f598230..6b89529a4 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -319,6 +319,7 @@ async def _emit_event(self, event, **kwargs): f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' ) for child_event in dns_child_events: + log.debug(f"Queueing DNS child for {event}: {child_event}") self.queue_event(child_event) except ValidationError as e: From db5c768a9259f63c0a4ec8d2adc0d1bcb9ef95e1 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 14 Feb 2024 11:42:48 -0500 Subject: [PATCH 012/171] steady work on presets --- bbot/cli.py | 8 +++++ bbot/core/config/args.py | 4 --- bbot/core/config/environ.py | 24 +++++++------- bbot/core/config/files.py | 7 +++-- bbot/core/config/logger.py | 9 +++--- bbot/core/core.py | 63 +++++++++++++++++++++++++++++++++---- bbot/core/modules.py | 52 +++++++++++++++--------------- 7 files changed, 112 insertions(+), 55 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 477c6a98b..14bdfbc4c 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -32,6 +32,13 @@ async def _main(): CORE.cli_execution = True + log.hugesuccess(CORE.default_config) + # log.hugeinfo(CORE.custom_config) + # log.hugewarning(CORE.module_loader.configs()) + # log.hugesuccess(CORE.default_config) + + return + # async def monitor_tasks(): # in_row = 0 # while 1: @@ -57,6 +64,7 @@ async def _main(): # monitor_tasks_task = asyncio.create_task(monitor_tasks()) try: + if len(sys.argv) == 1: CORE.args.parser.print_help() sys.exit(1) diff --git a/bbot/core/config/args.py b/bbot/core/config/args.py index f36238b87..64a555764 100644 --- a/bbot/core/config/args.py +++ b/bbot/core/config/args.py @@ -104,10 +104,6 @@ def __init__(self, core): self._flag_choices.update(set(c.get("flags", []))) self._flag_choices = sorted(self._flag_choices) - log_to_stderr(f"module options: {self._module_choices}") - log_to_stderr(f"output module options: {self._output_module_choices}") - log_to_stderr(f"flag options: {self._flag_choices}") - self.parser = self.create_parser() @property diff --git a/bbot/core/config/environ.py b/bbot/core/config/environ.py index 4d6233232..e94e1fad2 100644 --- a/bbot/core/config/environ.py +++ b/bbot/core/config/environ.py @@ -59,9 +59,9 @@ def prepare(self): """ # ensure bbot_home if not "home" in self.core.config: - self.core.config["home"] = "~/.bbot" - home = Path(self.core.config["home"]).expanduser().resolve() - self.core.config["home"] = str(home) + self.core.custom_config["home"] = "~/.bbot" + home = Path(self.core.custom_config["home"]).expanduser().resolve() + self.core.custom_config["home"] = str(home) # if we're running in a virtual environment, make sure to include its /bin in PATH if sys.prefix != sys.base_prefix: @@ -98,23 +98,23 @@ def prepare(self): # PRESET TODO: do we even need this? if self.core.cli_execution and self.core.args.cli_config: # deps - self.core.config["retry_deps"] = self.core.args.cli_options.retry_deps - self.core.config["force_deps"] = self.core.args.cli_options.force_deps - self.core.config["no_deps"] = self.core.args.cli_options.no_deps - self.core.config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps + self.core.custom_config["retry_deps"] = self.core.args.cli_options.retry_deps + self.core.custom_config["force_deps"] = self.core.args.cli_options.force_deps + self.core.custom_config["no_deps"] = self.core.args.cli_options.no_deps + self.core.custom_config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps # debug - self.core.config["debug"] = self.core.args.cli_options.debug - self.core.config["silent"] = self.core.args.cli_options.silent + self.core.custom_config["debug"] = self.core.args.cli_options.debug + self.core.custom_config["silent"] = self.core.args.cli_options.silent import logging log = logging.getLogger() - if self.core.config.get("debug", False): - self.core.config["silent"] = False + if self.core.custom_config.get("debug", False): + self.core.custom_config["silent"] = False log = logging.getLogger("bbot") log.setLevel(logging.DEBUG) logging.getLogger("asyncio").setLevel(logging.DEBUG) - elif self.core.config.get("silent", False): + elif self.core.custom_config.get("silent", False): log = logging.getLogger("bbot") log.setLevel(logging.CRITICAL) diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 03a227c4c..79c3521aa 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -75,10 +75,11 @@ def _get_config(self, filename, name="config"): raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") return OmegaConf.create() - def get_config(self): - default_config = self._get_config(self.defaults_filename, name="defaults") + def get_custom_config(self): return OmegaConf.merge( - default_config, self._get_config(self.config_filename, name="config"), self._get_config(self.secrets_filename, name="secrets"), ) + + def get_default_config(self): + return self._get_config(self.defaults_filename, name="defaults") diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 5a9f9bdf0..9e2ed8c2b 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -201,10 +201,11 @@ def log_level(self): return logging.DEBUG loglevel = logging.INFO - if self.core.args.parsed.verbose: - loglevel = logging.VERBOSE - if self.core.args.parsed.debug: - loglevel = logging.DEBUG + if self.core.cli_execution: + if self.core.args.parsed.verbose: + loglevel = logging.VERBOSE + if self.core.args.parsed.debug: + loglevel = logging.DEBUG return loglevel def set_log_level(self, level, logger=None): diff --git a/bbot/core/core.py b/bbot/core/core.py index a6300cfa4..041373bd0 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,4 +1,5 @@ from pathlib import Path +from omegaconf import OmegaConf class BBOTCore: @@ -15,6 +16,10 @@ def __init__(self): self._config = None self._default_config = None + self._custom_config = None + + # bare minimum == logging + self.logger # PRESET TODO: add back in bbot/core/configurator/__init__.py # - check_cli_args @@ -49,26 +54,60 @@ def files_config(self): @property def config(self): - if self._config is None: - from .config.logger import BBOTLogger + """ + .config is just .default_config + .custom_config merged together - self._config = self.files_config.get_config() - self._default_config = self._config.copy() - self._logger = BBOTLogger(self) + any new values should be added to custom_config. + """ + if self._config is None: + self._config = OmegaConf.merge(self.default_config, self.custom_config) + # set read-only flag (change .custom_config instead) + OmegaConf.set_readonly(self._config, True) return self._config @property def default_config(self): - self.config + if self._default_config is None: + self._default_config = self.files_config.get_default_config() + # set read-only flag (change .custom_config instead) + OmegaConf.set_readonly(self._default_config, True) return self._default_config + @default_config.setter + def default_config(self, value): + # we temporarily clear out the config so it can be refreshed if/when default_config changes + self._config = None + self._default_config = value + + @property + def custom_config(self): + # we temporarily clear out the config so it can be refreshed if/when custom_config changes + self._config = None + if self._custom_config is None: + self._custom_config = self.files_config.get_custom_config() + return self._custom_config + + @custom_config.setter + def custom_config(self, value): + # we temporarily clear out the config so it can be refreshed if/when custom_config changes + self._config = None + self._custom_config = value + @property def logger(self): self.config + if self._logger is None: + from .config.logger import BBOTLogger + + self._logger = BBOTLogger(self) return self._logger @property def module_loader(self): + # module loader depends on environment to be set up + # or is it the other way around + # PRESET TODO + self.environ if self._module_loader is None: from .modules import ModuleLoader @@ -79,6 +118,18 @@ def module_loader(self): self._module_loader = ModuleLoader(module_dirs=module_dirs) + # update default config with module defaults + module_config = OmegaConf.create( + { + "modules": self._module_loader.configs(type="scan"), + "output_modules": self._module_loader.configs(type="output"), + "internal_modules": self._module_loader.configs(type="internal"), + } + ) + # self.default_config = module_config + self.default_config = OmegaConf.merge(self.default_config, module_config) + assert False + return self._module_loader @property diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 1b3320535..38e82eba3 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -28,7 +28,7 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") def __init__(self, module_dirs=None): - self.__preloaded = {} + self.__preloaded = None self._preloaded_orig = None self._modules = {} self._configs = {} @@ -68,36 +68,36 @@ def preload(self): ... } """ - for module_dir in self.module_dirs: - for module_file in list_files(module_dir, filter=self.file_filter): - if module_dir.name == "modules": - namespace = f"bbot.modules" - else: - namespace = f"bbot.modules.{module_dir.name}" - try: - preloaded = self.preload_module(module_file) - module_type = "scan" - if module_dir.name in ("output", "internal"): - module_type = str(module_dir.name) - elif module_dir.name not in ("modules"): - preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) - preloaded["type"] = module_type - preloaded["namespace"] = namespace - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_file.stem] = config - self.__preloaded[module_file.stem] = preloaded - except Exception: - log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") - log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") - sys.exit(1) + if self.__preloaded is None: + self.__preloaded = {} + for module_dir in self.module_dirs: + for module_file in list_files(module_dir, filter=self.file_filter): + if module_dir.name == "modules": + namespace = f"bbot.modules" + else: + namespace = f"bbot.modules.{module_dir.name}" + try: + preloaded = self.preload_module(module_file) + module_type = "scan" + if module_dir.name in ("output", "internal"): + module_type = str(module_dir.name) + elif module_dir.name not in ("modules"): + preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) + preloaded["type"] = module_type + preloaded["namespace"] = namespace + config = OmegaConf.create(preloaded.get("config", {})) + self._configs[module_file.stem] = config + self.__preloaded[module_file.stem] = preloaded + except Exception: + log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") + log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") + sys.exit(1) return self.__preloaded @property def _preloaded(self): - if not self.__preloaded: - self.preload() - return self.__preloaded + return self.preload() def get_recursive_dirs(self, *dirs): dirs = set(Path(d) for d in dirs) From 34e5c9f28755b04c39fa7dd0277eee02b053f3d2 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 15 Feb 2024 17:31:06 -0500 Subject: [PATCH 013/171] preload cache --- bbot/cli.py | 2 +- bbot/core/config/environ.py | 14 +++------- bbot/core/core.py | 28 ++++++++++++++------ bbot/core/modules.py | 52 +++++++++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 62e88070d..8fb8d278e 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -36,7 +36,7 @@ async def _main(): # log.hugeinfo(CORE.custom_config) # log.hugewarning(CORE.module_loader.configs()) # log.hugesuccess(CORE.default_config) - print(CORE.module_loader.preloaded()) + print(CORE.module_loader) return diff --git a/bbot/core/config/environ.py b/bbot/core/config/environ.py index e94e1fad2..ee55651d3 100644 --- a/bbot/core/config/environ.py +++ b/bbot/core/config/environ.py @@ -57,12 +57,6 @@ def prepare(self): """ Sync config to OS environment variables """ - # ensure bbot_home - if not "home" in self.core.config: - self.core.custom_config["home"] = "~/.bbot" - home = Path(self.core.custom_config["home"]).expanduser().resolve() - self.core.custom_config["home"] = str(home) - # if we're running in a virtual environment, make sure to include its /bin in PATH if sys.prefix != sys.base_prefix: bin_dir = str(Path(sys.prefix) / "bin") @@ -73,18 +67,18 @@ def prepare(self): self.add_to_path(local_bin_dir) # ensure bbot_tools - bbot_tools = home / "tools" + bbot_tools = self.core.home / "tools" os.environ["BBOT_TOOLS"] = str(bbot_tools) if not str(bbot_tools) in os.environ.get("PATH", "").split(":"): os.environ["PATH"] = f'{bbot_tools}:{os.environ.get("PATH", "").strip(":")}' # ensure bbot_cache - bbot_cache = home / "cache" + bbot_cache = self.core.home / "cache" os.environ["BBOT_CACHE"] = str(bbot_cache) # ensure bbot_temp - bbot_temp = home / "temp" + bbot_temp = self.core.home / "temp" os.environ["BBOT_TEMP"] = str(bbot_temp) # ensure bbot_lib - bbot_lib = home / "lib" + bbot_lib = self.core.home / "lib" os.environ["BBOT_LIB"] = str(bbot_lib) # export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:~/.bbot/lib/ self.add_to_path(str(bbot_lib), k="LD_LIBRARY_PATH") diff --git a/bbot/core/core.py b/bbot/core/core.py index 041373bd0..1e810d204 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -2,8 +2,13 @@ from omegaconf import OmegaConf +bbot_code_dir = Path(__file__).parent.parent + + class BBOTCore: + default_module_dir = bbot_code_dir / "modules" + def __init__(self): self._args = None self._logger = None @@ -18,6 +23,19 @@ def __init__(self): self._default_config = None self._custom_config = None + # where to load modules from + self.module_dirs = self.config.get("module_dirs", []) + self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] + self.module_dirs = sorted(set(self.module_dirs)) + + # ensure bbot home dir + if not "home" in self.config: + self.custom_config["home"] = "~/.bbot" + self.home = Path(self.config["home"]).expanduser().resolve() + self.cache_dir = self.home / "cache" + self.tools_dir = self.home / "tools" + self.scans_dir = self.home / "scans" + # bare minimum == logging self.logger @@ -111,12 +129,7 @@ def module_loader(self): if self._module_loader is None: from .modules import ModuleLoader - # PRESET TODO: custom module load paths - module_dirs = self.config.get("module_dirs", []) - module_dirs = [Path(p) for p in module_dirs] - module_dirs = list(set(module_dirs)) - - self._module_loader = ModuleLoader(module_dirs=module_dirs) + self._module_loader = ModuleLoader(self) # update default config with module defaults module_config = OmegaConf.create( @@ -126,9 +139,8 @@ def module_loader(self): "internal_modules": self._module_loader.configs(type="internal"), } ) - # self.default_config = module_config + print(module_config) self.default_config = OmegaConf.merge(self.default_config, module_config) - assert False return self._module_loader diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 38e82eba3..a45110ab4 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -1,6 +1,7 @@ import re import ast import sys +import pickle import importlib import traceback from pathlib import Path @@ -12,9 +13,6 @@ from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform -bbot_code_dir = Path(__file__).parent.parent - - class ModuleLoader: """ Main class responsible for loading BBOT modules. @@ -24,22 +22,20 @@ class ModuleLoader: This ensures that all requisite libraries and components are available for the module to function correctly. """ - default_module_dir = bbot_code_dir / "modules" module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") - def __init__(self, module_dirs=None): + def __init__(self, core): + self.core = core self.__preloaded = None self._preloaded_orig = None self._modules = {} self._configs = {} - # start with default module dir - self.module_dirs = {self.default_module_dir} - # include custom ones if requested - if module_dirs: - self.module_dirs.update(set(Path(p) for p in module_dirs)) + self.preload_cache_file = self.core.cache_dir / "preloaded" + self._preload_cache = None + # expand to include all recursive dirs - self.module_dirs.update(self.get_recursive_dirs(*self.module_dirs)) + self.module_dirs = self.get_recursive_dirs(*self.core.module_dirs) def file_filter(self, file): file = file.resolve() @@ -72,6 +68,16 @@ def preload(self): self.__preloaded = {} for module_dir in self.module_dirs: for module_file in list_files(module_dir, filter=self.file_filter): + module_name = module_file.stem + module_file = module_file.resolve() + + # try to load from cache + module_cache_key = (str(module_file), tuple(module_file.stat())) + cache_key = self.preload_cache.get(module_name, {}).get("cache_key", ()) + if module_cache_key == cache_key: + self.__preloaded[module_name] = self.preload_cache[module_name] + continue + if module_dir.name == "modules": namespace = f"bbot.modules" else: @@ -85,16 +91,34 @@ def preload(self): preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) preloaded["type"] = module_type preloaded["namespace"] = namespace + preloaded["cache_key"] = module_cache_key config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_file.stem] = config - self.__preloaded[module_file.stem] = preloaded + self._configs[module_name] = config + self.__preloaded[module_name] = preloaded except Exception: log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") sys.exit(1) + self.preload_cache = self.__preloaded return self.__preloaded + @property + def preload_cache(self): + if self._preload_cache is None: + self._preload_cache = {} + if self.preload_cache_file.is_file(): + with suppress(Exception): + with open(self.preload_cache_file, "rb") as f: + self._preload_cache = pickle.load(f) + return self._preload_cache + + @preload_cache.setter + def preload_cache(self, value): + self._preload_cache = value + with open(self.preload_cache_file, "wb") as f: + pickle.dump(self._preload_cache, f) + @property def _preloaded(self): return self.preload() @@ -118,6 +142,7 @@ def preloaded(self, type=None): return preloaded def configs(self, type=None): + self.preload() configs = {} if type is not None: configs = {k: v for k, v in self._configs.items() if self.check_type(k, type)} @@ -271,6 +296,7 @@ def preload_module(self, module_file): "ansible": ansible_tasks, }, "sudo": len(apt_deps) > 0, + "code": python_code, } if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any( x == True for x in search_dict_by_key("ansible_become", ansible_tasks) From 101bc3c87f436208fb497bb7e41d5ca45f4d94cd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 15 Feb 2024 18:18:58 -0500 Subject: [PATCH 014/171] reduce global imports to speed up start time --- bbot/core/helpers/misc.py | 74 ++++++++++++++++++++++++++++----------- bbot/core/modules.py | 50 ++++++++++++++------------ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index a67e89402..f7dffade6 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1,35 +1,15 @@ import os import re import sys -import copy -import idna import json -import atexit -import codecs -import psutil import random -import shutil -import signal import string import asyncio -import difflib -import inspect import logging -import platform import ipaddress -import traceback import subprocess as sp from pathlib import Path -from itertools import islice -from datetime import datetime -from tabulate import tabulate -import wordninja as _wordninja from contextlib import suppress -import cloudcheck as _cloudcheck -import tldextract as _tldextract -import xml.etree.ElementTree as ET -from collections.abc import Mapping -from hashlib import sha1 as hashlib_sha1 from asyncio import create_task, gather, sleep, wait_for # noqa from urllib.parse import urlparse, quote, unquote, urlunparse # noqa F401 @@ -478,6 +458,8 @@ def tldextract(data): - Utilizes `smart_decode` to preprocess the data. - Makes use of the `tldextract` library for extraction. """ + import tldextract as _tldextract + return _tldextract.extract(smart_decode(data)) @@ -767,6 +749,8 @@ def sha1(data): >>> sha1("asdf").hexdigest() '3da541559918a808c2402bba5012f6c60b27661c' """ + from hashlib import sha1 as hashlib_sha1 + if isinstance(data, dict): data = json.dumps(data, sort_keys=True) return hashlib_sha1(smart_encode(data)) @@ -840,6 +824,8 @@ def recursive_decode(data, max_depth=5): >>> recursive_dcode("%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021") " Привет!" """ + import codecs + # Decode newline and tab escapes data = backslash_regex.sub( lambda match: {"n": "\n", "t": "\t", "r": "\r", "b": "\b", "v": "\v"}.get(match.group("char")), data @@ -954,6 +940,8 @@ def extract_params_xml(xml_data): >>> extract_params_xml('') {'child1', 'child2', 'root'} """ + import xml.etree.ElementTree as ET + try: root = ET.fromstring(xml_data) except ET.ParseError: @@ -1046,6 +1034,7 @@ def extract_words(data, acronyms=True, wordninja=True, model=None, max_length=10 >>> extract_words('blacklanternsecurity') {'black', 'lantern', 'security', 'bls', 'blacklanternsecurity'} """ + import wordninja as _wordninja if word_regexes is None: word_regexes = bbot_regexes.word_regexes @@ -1102,6 +1091,8 @@ def closest_match(s, choices, n=1, cutoff=0.0): >>> closest_match("asdf", ["asd", "fds", "asdff"], n=3) ['asdff', 'asd', 'fds'] """ + import difflib + matches = difflib.get_close_matches(s, choices, n=n, cutoff=cutoff) if not choices or not matches: return @@ -1136,10 +1127,16 @@ def match_and_exit(s, choices, msg=None, loglevel="HUGEWARNING", exitcode=2): sys.exit(2) -def kill_children(parent_pid=None, sig=signal.SIGTERM): +def kill_children(parent_pid=None, sig=None): """ Forgive me father for I have sinned """ + import psutil + import signal + + if sig is None: + sig = signal.SIGTERM + try: parent = psutil.Process(parent_pid) except psutil.NoSuchProcess: @@ -1262,6 +1259,8 @@ def rm_at_exit(path): Examples: >>> rm_at_exit("/tmp/test/file1.txt") """ + import atexit + atexit.register(delete_file, path) @@ -1375,6 +1374,8 @@ def which(*executables): >>> which("python", "python3") "/usr/bin/python" """ + import shutil + for e in executables: location = shutil.which(e) if location: @@ -1493,6 +1494,8 @@ def filter_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): >>> filter_dict({"key1": "test", "key2": {"key3": "asdf"}}, "key1", "key3", exclude_keys="key2") {'key1': 'test'} """ + import copy + if exclude_keys is None: exclude_keys = [] if isinstance(exclude_keys, str): @@ -1526,6 +1529,8 @@ def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): dict: A dictionary cleaned of the keys specified in key_names. """ + import copy + if exclude_keys is None: exclude_keys = [] if isinstance(exclude_keys, str): @@ -1556,6 +1561,7 @@ def grouper(iterable, n): >>> list(grouper('ABCDEFG', 3)) [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']] """ + from itertools import islice iterable = iter(iterable) return iter(lambda: list(islice(iterable, n)), []) @@ -1633,6 +1639,8 @@ def make_date(d=None, microseconds=False): >>> make_date(microseconds=True) "20220707_1330_35167617" """ + from datetime import datetime + f = "%Y%m%d_%H%M_%S" if microseconds: f += "%f" @@ -1766,6 +1774,8 @@ def rm_rf(f): Examples: >>> rm_rf("/tmp/httpx98323849") """ + import shutil + shutil.rmtree(f) @@ -1885,6 +1895,8 @@ def smart_encode_punycode(text: str) -> str: """ ドメイン.テスト --> xn--eckwd4c7c.xn--zckzah """ + import idna + host, before, after = extract_host(text) if host is None: return text @@ -1901,6 +1913,8 @@ def smart_decode_punycode(text: str) -> str: """ xn--eckwd4c7c.xn--zckzah --> ドメイン.テスト """ + import idna + host, before, after = extract_host(text) if host is None: return text @@ -1992,6 +2006,8 @@ def make_table(*args, **kwargs): | row2 | row2 | +-----------+-----------+ """ + from tabulate import tabulate + # fix IndexError: list index out of range if args and not args[0]: args = ([[]],) + args[1:] @@ -2134,6 +2150,8 @@ def cpu_architecture(): >>> cpu_architecture() 'amd64' """ + import platform + uname = platform.uname() arch = uname.machine.lower() if arch.startswith("aarch"): @@ -2224,6 +2242,8 @@ def memory_status(): >>> mem.percent 79.0 """ + import psutil + return psutil.virtual_memory() @@ -2246,6 +2266,8 @@ def swap_status(): >>> swap.used 2097152 """ + import psutil + return psutil.swap_memory() @@ -2268,6 +2290,8 @@ def get_size(obj, max_depth=5, seen=None): >>> get_size(my_dict, max_depth=3) 8400 """ + from collections.abc import Mapping + # If seen is not provided, initialize an empty set if seen is None: seen = set() @@ -2343,6 +2367,8 @@ def cloudcheck(ip): >>> cloudcheck("168.62.20.37") ('Azure', 'cloud', IPv4Network('168.62.0.0/19')) """ + import cloudcheck as _cloudcheck + return _cloudcheck.check(ip) @@ -2362,6 +2388,8 @@ def is_async_function(f): >>> is_async_function(foo) True """ + import inspect + return inspect.iscoroutinefunction(f) @@ -2440,6 +2468,8 @@ def get_traceback_details(e): ... print(f"File: {filename}, Line: {lineno}, Function: {funcname}") File: , Line: 2, Function: """ + import traceback + tb = traceback.extract_tb(e.__traceback__) last_frame = tb[-1] # Get the last frame in the traceback (the one where the exception was raised) filename = last_frame.filename @@ -2478,6 +2508,8 @@ async def cancel_tasks(tasks, ignore_errors=True): await task except BaseException as e: if not isinstance(e, asyncio.CancelledError): + import traceback + log.trace(traceback.format_exc()) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index a45110ab4..7ae4082b8 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -75,30 +75,34 @@ def preload(self): module_cache_key = (str(module_file), tuple(module_file.stat())) cache_key = self.preload_cache.get(module_name, {}).get("cache_key", ()) if module_cache_key == cache_key: - self.__preloaded[module_name] = self.preload_cache[module_name] - continue - - if module_dir.name == "modules": - namespace = f"bbot.modules" + preloaded = self.preload_cache[module_name] else: - namespace = f"bbot.modules.{module_dir.name}" - try: - preloaded = self.preload_module(module_file) - module_type = "scan" - if module_dir.name in ("output", "internal"): - module_type = str(module_dir.name) - elif module_dir.name not in ("modules"): - preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) - preloaded["type"] = module_type - preloaded["namespace"] = namespace - preloaded["cache_key"] = module_cache_key - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_name] = config - self.__preloaded[module_name] = preloaded - except Exception: - log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") - log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") - sys.exit(1) + if module_dir.name == "modules": + namespace = f"bbot.modules" + else: + namespace = f"bbot.modules.{module_dir.name}" + try: + preloaded = self.preload_module(module_file) + module_type = "scan" + if module_dir.name in ("output", "internal"): + module_type = str(module_dir.name) + elif module_dir.name not in ("modules"): + preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) + preloaded["type"] = module_type + preloaded["namespace"] = namespace + preloaded["cache_key"] = module_cache_key + + except Exception: + log_to_stderr( + f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL" + ) + log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") + sys.exit(1) + + self.__preloaded[module_name] = preloaded + config = OmegaConf.create(preloaded.get("config", {})) + self._configs[module_name] = config + self.preload_cache = self.__preloaded return self.__preloaded From 67808762204d63636880208d7ccfd7e49d186f08 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 15 Feb 2024 18:26:58 -0500 Subject: [PATCH 015/171] defer imports when possible --- bbot/core/errors.py | 3 --- bbot/modules/output/http.py | 2 +- bbot/scanner/scanner.py | 4 +++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bbot/core/errors.py b/bbot/core/errors.py index 5e5f57aeb..d86899db2 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -1,6 +1,3 @@ -from httpx import HTTPError, RequestError # noqa - - class BBOTError(Exception): pass diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 10ca1c8df..719bf40c4 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,4 +1,4 @@ -from bbot.core.errors import RequestError +from httpx import RequestError from bbot.modules.output.base import BaseOutputModule diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a21af0e2a..322a74d1e 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -23,7 +23,6 @@ from .dispatcher import Dispatcher from bbot.core.event import make_event from bbot.core.helpers.misc import sha1, rand_string -from bbot.core.helpers.helper import ConfigAwareHelper from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen from bbot.core.errors import BBOTError, ScanError, ValidationError @@ -167,6 +166,9 @@ def __init__( self._status_code = 0 self.max_workers = max(1, self.config.get("max_threads", 25)) + + from bbot.core.helpers.helper import ConfigAwareHelper + self.helpers = ConfigAwareHelper(config=self.config, scan=self) if name is None: From 30eb11ddc9b33d29e3741acf33d2094695750b74 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 16 Feb 2024 11:29:23 -0500 Subject: [PATCH 016/171] remove debug statement --- bbot/core/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index 1e810d204..c1bad122b 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -139,7 +139,6 @@ def module_loader(self): "internal_modules": self._module_loader.configs(type="internal"), } ) - print(module_config) self.default_config = OmegaConf.merge(self.default_config, module_config) return self._module_loader From e19edae492893d359d4a50f3fe6fef7f0d727d63 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 23 Feb 2024 11:45:46 -0500 Subject: [PATCH 017/171] laying the groundwork for presets --- bbot/cli.py | 5 ++ bbot/core/config/logger.py | 11 +-- bbot/core/core.py | 53 ++--------- bbot/core/modules.py | 8 +- bbot/{core/config => scanner/preset}/args.py | 15 ++-- .../config => scanner/preset}/environ.py | 21 ++--- bbot/scanner/preset/preset.py | 90 +++++++++++++++++++ bbot/scanner/scanner.py | 26 ++---- 8 files changed, 135 insertions(+), 94 deletions(-) rename bbot/{core/config => scanner/preset}/args.py (95%) rename bbot/{core/config => scanner/preset}/environ.py (86%) create mode 100644 bbot/scanner/preset/preset.py diff --git a/bbot/cli.py b/bbot/cli.py index 8fb8d278e..367baa6cd 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -38,6 +38,11 @@ async def _main(): # log.hugesuccess(CORE.default_config) print(CORE.module_loader) + # 1) load core (environment variables, modules) + # 2) instantiate god preset + # 3) pull in command line arguments (this adds targets, configs, presets, etc.) + # 4) pass preset to scan + return # async def monitor_tasks(): diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 9e2ed8c2b..e3ea245ba 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -201,11 +201,12 @@ def log_level(self): return logging.DEBUG loglevel = logging.INFO - if self.core.cli_execution: - if self.core.args.parsed.verbose: - loglevel = logging.VERBOSE - if self.core.args.parsed.debug: - loglevel = logging.DEBUG + # PRESET TODO: delete / move this + # if self.core.cli_execution: + # if self.core.args.parsed.verbose: + # loglevel = logging.VERBOSE + # if self.core.args.parsed.debug: + # loglevel = logging.DEBUG return loglevel def set_log_level(self, level, logger=None): diff --git a/bbot/core/core.py b/bbot/core/core.py index c1bad122b..c9a167670 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -10,11 +10,8 @@ class BBOTCore: default_module_dir = bbot_code_dir / "modules" def __init__(self): - self._args = None self._logger = None - self._environ = None self._files_config = None - self._module_loader = None self.bbot_sudo_pass = None self.cli_execution = False @@ -23,11 +20,6 @@ def __init__(self): self._default_config = None self._custom_config = None - # where to load modules from - self.module_dirs = self.config.get("module_dirs", []) - self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] - self.module_dirs = sorted(set(self.module_dirs)) - # ensure bbot home dir if not "home" in self.config: self.custom_config["home"] = "~/.bbot" @@ -111,6 +103,12 @@ def custom_config(self, value): self._config = None self._custom_config = value + def merge_custom(self, config): + self.custom_config = Omegaconf.merge(self.custom_config, OmegaConf.create(config)) + + def merge_default(self, config): + self.default_config = Omegaconf.merge(self.default_config, OmegaConf.create(config)) + @property def logger(self): self.config @@ -119,42 +117,3 @@ def logger(self): self._logger = BBOTLogger(self) return self._logger - - @property - def module_loader(self): - # module loader depends on environment to be set up - # or is it the other way around - # PRESET TODO - self.environ - if self._module_loader is None: - from .modules import ModuleLoader - - self._module_loader = ModuleLoader(self) - - # update default config with module defaults - module_config = OmegaConf.create( - { - "modules": self._module_loader.configs(type="scan"), - "output_modules": self._module_loader.configs(type="output"), - "internal_modules": self._module_loader.configs(type="internal"), - } - ) - self.default_config = OmegaConf.merge(self.default_config, module_config) - - return self._module_loader - - @property - def environ(self): - if self._environ is None: - from .config.environ import BBOTEnviron - - self._environ = BBOTEnviron(self) - return self._environ - - @property - def args(self): - if self._args is None: - from .config.args import BBOTArgs - - self._args = BBOTArgs(self) - return self._args diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 7ae4082b8..4e40bde97 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -24,18 +24,18 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") - def __init__(self, core): - self.core = core + def __init__(self, preset): + self.preset = preset self.__preloaded = None self._preloaded_orig = None self._modules = {} self._configs = {} - self.preload_cache_file = self.core.cache_dir / "preloaded" + self.preload_cache_file = self.preset.core.cache_dir / "preloaded" self._preload_cache = None # expand to include all recursive dirs - self.module_dirs = self.get_recursive_dirs(*self.core.module_dirs) + self.module_dirs = self.get_recursive_dirs(*self.preset.module_dirs) def file_filter(self, file): file = file.resolve() diff --git a/bbot/core/config/args.py b/bbot/scanner/preset/args.py similarity index 95% rename from bbot/core/config/args.py rename to bbot/scanner/preset/args.py index 64a555764..fa7e3071e 100644 --- a/bbot/core/config/args.py +++ b/bbot/scanner/preset/args.py @@ -92,15 +92,16 @@ class BBOTArgs: for title, description, command in example: epilog += f"\n {title}:\n {command}\n" - def __init__(self, core): - self.core = core + def __init__(self, preset): + self.preset = preset self._parsed = None self._cli_config = None - self._module_choices = sorted(set(self.core.module_loader.preloaded(type="scan"))) - self._output_module_choices = sorted(set(self.core.module_loader.preloaded(type="output"))) + # PRESET TODO: revisit this + self._module_choices = sorted(set(self.preset.module_loader.preloaded(type="scan"))) + self._output_module_choices = sorted(set(self.preset.module_loader.preloaded(type="output"))) self._flag_choices = set() - for m, c in self.core.module_loader.preloaded().items(): + for m, c in self.preset.module_loader.preloaded().items(): self._flag_choices.update(set(c.get("flags", []))) self._flag_choices = sorted(self._flag_choices) @@ -280,7 +281,7 @@ def validate(self): all_options = None for c in conf: c = c.split("=")[0].strip() - v = OmegaConf.select(self.core.default_config, c, default=sentinel) + v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel) # if option isn't in the default config if v is sentinel: if self.exclude_from_validation.match(c): @@ -291,6 +292,6 @@ def validate(self): modules_options = set() for module_options in module_loader.modules_options().values(): modules_options.update(set(o[0] for o in module_options)) - global_options = set(self.core.default_config.keys()) - {"modules", "output_modules"} + global_options = set(self.preset.core.default_config.keys()) - {"modules", "output_modules"} all_options = global_options.union(modules_options) match_and_exit(c, all_options, msg="module option") diff --git a/bbot/core/config/environ.py b/bbot/scanner/preset/environ.py similarity index 86% rename from bbot/core/config/environ.py rename to bbot/scanner/preset/environ.py index ee55651d3..de867de70 100644 --- a/bbot/core/config/environ.py +++ b/bbot/scanner/preset/environ.py @@ -90,15 +90,15 @@ def prepare(self): # exchange certain options between CLI args and config # PRESET TODO: do we even need this? - if self.core.cli_execution and self.core.args.cli_config: - # deps - self.core.custom_config["retry_deps"] = self.core.args.cli_options.retry_deps - self.core.custom_config["force_deps"] = self.core.args.cli_options.force_deps - self.core.custom_config["no_deps"] = self.core.args.cli_options.no_deps - self.core.custom_config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps - # debug - self.core.custom_config["debug"] = self.core.args.cli_options.debug - self.core.custom_config["silent"] = self.core.args.cli_options.silent + # if self.core.cli_execution and self.core.args.cli_config: + # # deps + # self.core.custom_config["retry_deps"] = self.core.args.cli_options.retry_deps + # self.core.custom_config["force_deps"] = self.core.args.cli_options.force_deps + # self.core.custom_config["no_deps"] = self.core.args.cli_options.no_deps + # self.core.custom_config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps + # # debug + # self.core.custom_config["debug"] = self.core.args.cli_options.debug + # self.core.custom_config["silent"] = self.core.args.cli_options.silent import logging @@ -126,7 +126,8 @@ def prepare(self): os.environ.pop("HTTPS_PROXY", None) # replace environment variables in preloaded modules - self.core.module_loader.find_and_replace(**os.environ) + # PRESET TODO: move this + # self.core.module_loader.find_and_replace(**os.environ) # ssl verification import urllib3 diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py new file mode 100644 index 000000000..2ddb37f47 --- /dev/null +++ b/bbot/scanner/preset/preset.py @@ -0,0 +1,90 @@ +import copy +from omegaconf import OmegaConf + +from bbot.core import CORE + + +bbot_code_dir = Path(__file__).parent.parent.parent + + +class Preset: + """ + CORE should not handle arguments + which means it shouldn't need access to the module loader + + """ + + default_module_dir = bbot_code_dir / "modules" + + def __init__( + self, + *targets, + whitelist=None, + blacklist=None, + scan_id=None, + name=None, + modules=None, + output_modules=None, + output_dir=None, + config=None, + dispatcher=None, + strict_scope=False, + ): + + self._args = None + self._environ = None + self._module_loader = None + + # where to load modules from + self.module_dirs = self.config.get("module_dirs", []) + self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] + self.module_dirs = sorted(set(self.module_dirs)) + + # make a copy of BBOT core + self.core = copy.deepcopy(CORE) + if config is None: + config = OmegaConf.create({}) + # merge any custom configs + self.core.config.merge_custom(config) + + def process_cli_args(self): + pass + + @property + def module_loader(self): + # module loader depends on environment to be set up + # or is it the other way around + # PRESET TODO + self.environ + if self._module_loader is None: + from bbot.core.modules import ModuleLoader + + self._module_loader = ModuleLoader(self) + + # update default config with module defaults + module_config = OmegaConf.create( + { + "modules": self._module_loader.configs(type="scan"), + "output_modules": self._module_loader.configs(type="output"), + "internal_modules": self._module_loader.configs(type="internal"), + } + ) + self.core.merge_default(module_config) + + return self._module_loader + + @property + def environ(self): + if self._environ is None: + from .config.environ import BBOTEnviron + + self._environ = BBOTEnviron(self) + return self._environ + + @property + def args(self): + if self._args is None: + from .args import BBOTArgs + + self._args = BBOTArgs(self) + return self._args diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 322a74d1e..ce8c6ba6b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -19,6 +19,7 @@ from .target import Target from .stats import ScanStats +from bbot.presets import Preset from .manager import ScanManager from .dispatcher import Dispatcher from bbot.core.event import make_event @@ -102,21 +103,7 @@ class Scanner: "FINISHED": 8, } - def __init__( - self, - *targets, - whitelist=None, - blacklist=None, - scan_id=None, - name=None, - modules=None, - output_modules=None, - output_dir=None, - config=None, - dispatcher=None, - strict_scope=False, - force_start=False, - ): + def __init__(self, *targets, force_start=False, **preset_kwargs): """ Initializes the Scanner class. @@ -134,6 +121,9 @@ def __init__( strict_scope (bool, optional): If True, only targets explicitly in whitelist are scanned. Defaults to False. force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. """ + + self.preset = Preset(*targets, **preset_kwargs) + if modules is None: modules = [] if output_modules is None: @@ -144,12 +134,6 @@ def __init__( if isinstance(output_modules, str): output_modules = [output_modules] - if config is None: - config = OmegaConf.create({}) - else: - config = OmegaConf.create(config) - self.config = OmegaConf.merge(CORE.config, config) - # PRESET TODO: revisit this CORE.environ.prepare() if self.config.get("debug", False): From c867853711995cb439ff622c3a80b763c30b9ee6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 26 Feb 2024 16:23:59 -0500 Subject: [PATCH 018/171] steady work on presets --- bbot/cli.py | 34 +--- bbot/core/config/logger.py | 4 +- bbot/core/core.py | 9 +- bbot/core/helpers/depsinstaller/installer.py | 17 +- bbot/core/helpers/helper.py | 15 +- bbot/core/helpers/misc.py | 2 + bbot/core/helpers/web.py | 17 +- bbot/scanner/preset/__init__.py | 1 + bbot/scanner/preset/environ.py | 2 +- bbot/scanner/preset/preset.py | 113 +++++++++++-- bbot/scanner/scanner.py | 160 ++++++++----------- bbot/scanner/target.py | 15 +- bbot/scripts/docs.py | 2 +- 13 files changed, 219 insertions(+), 172 deletions(-) create mode 100644 bbot/scanner/preset/__init__.py diff --git a/bbot/cli.py b/bbot/cli.py index 367baa6cd..821b6c9ab 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -15,6 +15,8 @@ from bbot.core import CORE +CORE.logger.set_log_level("DEBUG") + from bbot.core import errors from bbot import __version__ from bbot.core.helpers.misc import smart_decode @@ -30,13 +32,17 @@ async def _main(): global err global scan_name - CORE.cli_execution = True + from bbot.scanner import Scanner + + scan = Scanner("www.example.com", _cli_execution=True) + async for event in scan.async_start(): + print(event) # log.hugesuccess(CORE.default_config) # log.hugeinfo(CORE.custom_config) # log.hugewarning(CORE.module_loader.configs()) # log.hugesuccess(CORE.default_config) - print(CORE.module_loader) + # print(CORE.module_loader) # 1) load core (environment variables, modules) # 2) instantiate god preset @@ -45,30 +51,6 @@ async def _main(): return - # async def monitor_tasks(): - # in_row = 0 - # while 1: - # try: - # print('looooping') - # tasks = asyncio.all_tasks() - # current_task = asyncio.current_task() - # if len(tasks) == 1 and list(tasks)[0] == current_task: - # print('no tasks') - # in_row += 1 - # else: - # in_row = 0 - # for t in tasks: - # print(t) - # if in_row > 2: - # break - # await asyncio.sleep(1) - # except BaseException as e: - # print(traceback.format_exc()) - # with suppress(BaseException): - # await asyncio.sleep(.1) - - # monitor_tasks_task = asyncio.create_task(monitor_tasks()) - try: if len(sys.argv) == 1: diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index e3ea245ba..d72a59395 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -210,9 +210,11 @@ def log_level(self): return loglevel def set_log_level(self, level, logger=None): + if isinstance(level, str): + level = logging.getLevelName(level) if logger is not None: logger.hugeinfo(f"Setting log level to {logging.getLevelName(level)}") - self.core.config["silent"] = False + self.core.custom_config["silent"] = False self._log_level_override = level for logger in self.loggers: logger.setLevel(level) diff --git a/bbot/core/core.py b/bbot/core/core.py index c9a167670..f88f2bfaf 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -2,13 +2,8 @@ from omegaconf import OmegaConf -bbot_code_dir = Path(__file__).parent.parent - - class BBOTCore: - default_module_dir = bbot_code_dir / "modules" - def __init__(self): self._logger = None self._files_config = None @@ -104,10 +99,10 @@ def custom_config(self, value): self._custom_config = value def merge_custom(self, config): - self.custom_config = Omegaconf.merge(self.custom_config, OmegaConf.create(config)) + self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) def merge_default(self, config): - self.default_config = Omegaconf.merge(self.default_config, OmegaConf.create(config)) + self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config)) @property def logger(self): diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index fbb9f2b59..fa32edf8b 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -13,8 +13,6 @@ from ansible_runner.interface import run from subprocess import CalledProcessError -from bbot.core import CORE - from ..misc import can_sudo_without_password, os_platform log = logging.getLogger("bbot.core.helpers.depsinstaller") @@ -23,6 +21,8 @@ class DepsInstaller: def __init__(self, parent_helper): self.parent_helper = parent_helper + self.preset = self.parent_helper.preset + self.core = self.preset.core # respect BBOT's http timeout http_timeout = self.parent_helper.config.get("http_timeout", 30) @@ -32,8 +32,8 @@ def __init__(self, parent_helper): self._installed_sudo_askpass = False self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None) if self._sudo_password is None: - if CORE.bbot_sudo_pass is not None: - self._sudo_password = CORE.bbot_sudo_pass + if self.core.bbot_sudo_pass is not None: + self._sudo_password = self.core.bbot_sudo_pass elif can_sudo_without_password(): self._sudo_password = "" self.data_dir = self.parent_helper.cache_dir / "depsinstaller" @@ -52,9 +52,6 @@ def __init__(self, parent_helper): if sys.prefix != sys.base_prefix: self.venv = sys.prefix - # PRESET TODO: revisit this - self.all_modules_preloaded = CORE.module_loader.preloaded() - self.ensure_root_lock = Lock() async def install(self, *modules): @@ -311,7 +308,7 @@ def ensure_root(self, message=""): if self.parent_helper.verify_sudo_password(password): log.success("Authentication successful") self._sudo_password = password - CORE.bbot_sudo_pass = password + self.core.bbot_sudo_pass = password else: log.warning("Incorrect password") @@ -337,3 +334,7 @@ def _install_sudo_askpass(self): askpass_dst = self.parent_helper.tools_dir / self.askpass_filename shutil.copy(askpass_src, askpass_dst) askpass_dst.chmod(askpass_dst.stat().st_mode | stat.S_IEXEC) + + @property + def all_modules_preloaded(self): + return self.preset.module_loader.preloaded() diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 899f3ab0b..9fa5eb1a5 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -51,9 +51,8 @@ class ConfigAwareHelper: from .cache import cache_get, cache_put, cache_filename, is_cached from .command import run, run_live, _spawn_proc, _prepare_command_kwargs - def __init__(self, config, scan=None): - self.config = config - self._scan = scan + def __init__(self, preset): + self.preset = preset self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() self.cache_dir = self.bbot_home / "cache" self.temp_dir = self.bbot_home / "temp" @@ -100,12 +99,12 @@ def make_target(self, *events): return Target(self.scan, *events) @property - def scan(self): - if self._scan is None: - from bbot.scanner import Scanner + def config(self): + return self.preset.core.config - self._scan = Scanner() - return self._scan + @property + def scan(self): + return self.preset.scan @property def in_tests(self): diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index f7dffade6..18eb43565 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2174,6 +2174,8 @@ def os_platform(): >>> os_platform() 'linux' """ + import platform + return platform.system().lower() diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 1a442c7e3..d754a53b7 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -50,18 +50,19 @@ class BBOTAsyncClient(httpx.AsyncClient): """ def __init__(self, *args, **kwargs): - self._bbot_scan = kwargs.pop("_bbot_scan") - web_requests_per_second = self._bbot_scan.config.get("web_requests_per_second", 100) + self._preset = kwargs.pop("_preset") + self._bbot_scan = self._preset.scan + web_requests_per_second = self._preset.config.get("web_requests_per_second", 100) self._rate_limiter = RateLimiter(web_requests_per_second, "Web") - http_debug = self._bbot_scan.config.get("http_debug", None) + http_debug = self._preset.config.get("http_debug", None) if http_debug: log.trace(f"Creating AsyncClient: {args}, {kwargs}") self._persist_cookies = kwargs.pop("persist_cookies", True) # timeout - http_timeout = self._bbot_scan.config.get("http_timeout", 20) + http_timeout = self._preset.config.get("http_timeout", 20) if not "timeout" in kwargs: kwargs["timeout"] = http_timeout @@ -70,12 +71,12 @@ def __init__(self, *args, **kwargs): if headers is None: headers = {} # user agent - user_agent = self._bbot_scan.config.get("user_agent", "BBOT") + user_agent = self._preset.config.get("user_agent", "BBOT") if "User-Agent" not in headers: headers["User-Agent"] = user_agent kwargs["headers"] = headers # proxy - proxies = self._bbot_scan.config.get("http_proxy", None) + proxies = self._preset.config.get("http_proxy", None) kwargs["proxies"] = proxies super().__init__(*args, **kwargs) @@ -90,7 +91,7 @@ def build_request(self, *args, **kwargs): request = super().build_request(*args, **kwargs) # add custom headers if the URL is in-scope if self._bbot_scan.in_scope(str(request.url)): - for hk, hv in self._bbot_scan.config.get("http_headers", {}).items(): + for hk, hv in self._preset.config.get("http_headers", {}).items(): # don't clobber headers if hk not in request.headers: request.headers[hk] = hv @@ -141,7 +142,7 @@ def __init__(self, parent_helper): self.web_client = self.AsyncClient(persist_cookies=False) def AsyncClient(self, *args, **kwargs): - kwargs["_bbot_scan"] = self.parent_helper.scan + kwargs["_preset"] = self.parent_helper.preset retries = kwargs.pop("retries", self.parent_helper.config.get("http_retries", 1)) kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify) kwargs["verify"] = self.ssl_verify diff --git a/bbot/scanner/preset/__init__.py b/bbot/scanner/preset/__init__.py new file mode 100644 index 000000000..a6fbc24bb --- /dev/null +++ b/bbot/scanner/preset/__init__.py @@ -0,0 +1 @@ +from .preset import Preset diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index de867de70..89d21987d 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -3,7 +3,7 @@ import omegaconf from pathlib import Path -from ..helpers.misc import cpu_architecture, os_platform, os_platform_friendly +from bbot.core.helpers.misc import cpu_architecture, os_platform, os_platform_friendly def increase_limit(new_limit): diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 2ddb37f47..3da0aef9e 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,7 +1,9 @@ -import copy +from pathlib import Path from omegaconf import OmegaConf from bbot.core import CORE +from bbot.core.helpers.misc import sha1, rand_string +from bbot.core.helpers.names_generator import random_name bbot_code_dir = Path(__file__).parent.parent.parent @@ -19,37 +21,118 @@ class Preset: def __init__( self, *targets, + scan=None, whitelist=None, blacklist=None, scan_id=None, - name=None, + scan_name=None, modules=None, output_modules=None, output_dir=None, config=None, dispatcher=None, strict_scope=False, + _cli_execution=False, ): + self._scan = scan self._args = None self._environ = None self._module_loader = None + self._cli_execution = _cli_execution - # where to load modules from - self.module_dirs = self.config.get("module_dirs", []) - self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] - self.module_dirs = sorted(set(self.module_dirs)) - - # make a copy of BBOT core - self.core = copy.deepcopy(CORE) + # bbot core config + self.core = CORE if config is None: config = OmegaConf.create({}) # merge any custom configs - self.core.config.merge_custom(config) + self.core.merge_custom(config) + + if modules is None: + modules = [] + if output_modules is None: + output_modules = ["python"] + if isinstance(modules, str): + modules = [modules] + if isinstance(output_modules, str): + output_modules = [output_modules] + self.scan_modules = modules + self.output_modules = output_modules + + # PRESET TODO: preparation of environment + # self.core.environ.prepare() + if self.core.config.get("debug", False): + self.core.logger.set_log_level("DEBUG") + + # dirs to load modules from + self.module_dirs = self.core.config.get("module_dirs", []) + self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] + self.module_dirs = sorted(set(self.module_dirs)) + + # config-aware helper + from bbot.core.helpers.helper import ConfigAwareHelper + + self.helpers = ConfigAwareHelper(preset=self) + + if scan_id is not None: + self.scan_id = str(scan_id) + else: + self.scan_id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" + + # scan name + if scan_name is None: + tries = 0 + while 1: + if tries > 5: + self.scan_name = f"{self.helpers.rand_string(4)}_{self.helpers.rand_string(4)}" + break + self.scan_name = random_name() + if output_dir is not None: + home_path = Path(output_dir).resolve() / self.scan_name + else: + home_path = self.helpers.bbot_home / "scans" / self.scan_name + if not home_path.exists(): + break + tries += 1 + else: + self.scan_name = str(scan_name) + + # scan output dir + if output_dir is not None: + self.scan_home = Path(output_dir).resolve() / self.scan_name + else: + self.scan_home = self.helpers.bbot_home / "scans" / self.scan_name + + def set_scope(self, targets, whitelist, blacklist, strict_scope=False): + self.strict_scope = strict_scope + + # target / whitelist / blacklist + from bbot.scanner.target import Target + + self.target = Target(self, *targets, strict_scope=strict_scope, make_in_scope=True) + if not whitelist: + self.whitelist = self.target.copy() + else: + self.whitelist = Target(self, *whitelist, strict_scope=self.strict_scope) + if not blacklist: + blacklist = [] + self.blacklist = Target(self, *blacklist) def process_cli_args(self): pass + @property + def config(self): + return self.core.config + + @property + def scan(self): + if self._scan is None: + from bbot.scanner import Scanner + + self._scan = Scanner() + return self._scan + @property def module_loader(self): # module loader depends on environment to be set up @@ -73,10 +156,18 @@ def module_loader(self): return self._module_loader + @property + def internal_modules(self): + return list(self.module_loader.preloaded(type="internal")) + + @property + def all_modules(self): + return sorted(set(self.scan_modules + self.output_modules + self.internal_modules)) + @property def environ(self): if self._environ is None: - from .config.environ import BBOTEnviron + from .environ import BBOTEnviron self._environ = BBOTEnviron(self) return self._environ diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index ce8c6ba6b..de345da62 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -5,26 +5,20 @@ import traceback import contextlib from sys import exc_info -from pathlib import Path import multiprocessing as mp from datetime import datetime from functools import partial -from omegaconf import OmegaConf from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor -from bbot.core import CORE from bbot import __version__ -from .target import Target +from .preset import Preset from .stats import ScanStats -from bbot.presets import Preset from .manager import ScanManager from .dispatcher import Dispatcher from bbot.core.event import make_event -from bbot.core.helpers.misc import sha1, rand_string -from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen from bbot.core.errors import BBOTError, ScanError, ValidationError @@ -103,7 +97,16 @@ class Scanner: "FINISHED": 8, } - def __init__(self, *targets, force_start=False, **preset_kwargs): + def __init__( + self, + *targets, + whitelist=None, + blacklist=None, + strict_scope=False, + dispatcher=None, + force_start=False, + **preset_kwargs, + ): """ Initializes the Scanner class. @@ -122,77 +125,18 @@ def __init__(self, *targets, force_start=False, **preset_kwargs): force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. """ - self.preset = Preset(*targets, **preset_kwargs) + self.preset = Preset(scan=self, **preset_kwargs) + self.preset.set_scope(targets, whitelist, blacklist, strict_scope=strict_scope) - if modules is None: - modules = [] - if output_modules is None: - output_modules = ["python"] - - if isinstance(modules, str): - modules = [modules] - if isinstance(output_modules, str): - output_modules = [output_modules] - - # PRESET TODO: revisit this - CORE.environ.prepare() - if self.config.get("debug", False): - CORE.logger.set_log_level(logging.DEBUG) - - self.strict_scope = strict_scope self.force_start = force_start - - if scan_id is not None: - self.id = str(scan_id) - else: - self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" self._status = "NOT_STARTED" self._status_code = 0 self.max_workers = max(1, self.config.get("max_threads", 25)) - from bbot.core.helpers.helper import ConfigAwareHelper - - self.helpers = ConfigAwareHelper(config=self.config, scan=self) - - if name is None: - tries = 0 - while 1: - if tries > 5: - self.name = f"{self.helpers.rand_string(4)}_{self.helpers.rand_string(4)}" - break - self.name = random_name() - if output_dir is not None: - home_path = Path(output_dir).resolve() / self.name - else: - home_path = self.helpers.bbot_home / "scans" / self.name - if not home_path.exists(): - break - tries += 1 - else: - self.name = str(name) - - if output_dir is not None: - self.home = Path(output_dir).resolve() / self.name - else: - self.home = self.helpers.bbot_home / "scans" / self.name - - self.target = Target(self, *targets, strict_scope=strict_scope, make_in_scope=True) - self.modules = OrderedDict({}) - self._scan_modules = modules - self._internal_modules = list(self._internal_modules()) - self._output_modules = output_modules self._modules_loaded = False - if not whitelist: - self.whitelist = self.target.copy() - else: - self.whitelist = Target(self, *whitelist, strict_scope=strict_scope) - if not blacklist: - blacklist = [] - self.blacklist = Target(self, *blacklist) - if dispatcher is None: self.dispatcher = Dispatcher() else: @@ -259,7 +203,7 @@ async def _prep(self): self.helpers.mkdir(self.home) if not self._prepped: - start_msg = f"Scan with {len(self._scan_modules):,} modules seeded with {len(self.target):,} targets" + start_msg = f"Scan with {len(self.preset.scan_modules):,} modules seeded with {len(self.target):,} targets" details = [] if self.whitelist != self.target: details.append(f"{len(self.whitelist):,} in whitelist") @@ -476,24 +420,21 @@ async def load_modules(self): After all modules are loaded, they are sorted by `_priority` and stored in the `modules` dictionary. """ if not self._modules_loaded: - all_modules = list(set(self._scan_modules + self._output_modules + self._internal_modules)) - if not all_modules: + if not self.preset.all_modules: self.warning(f"No modules to load") return - if not self._scan_modules: + if not self.preset.scan_modules: self.warning(f"No scan modules to load") # install module dependencies - succeeded, failed = await self.helpers.depsinstaller.install( - *self._scan_modules, *self._output_modules, *self._internal_modules - ) + succeeded, failed = await self.helpers.depsinstaller.install(*self.preset.all_modules) if failed: msg = f"Failed to install dependencies for {len(failed):,} modules: {','.join(failed)}" self._fail_setup(msg) - modules = sorted([m for m in self._scan_modules if m in succeeded]) - output_modules = sorted([m for m in self._output_modules if m in succeeded]) - internal_modules = sorted([m for m in self._internal_modules if m in succeeded]) + modules = sorted([m for m in self.preset.scan_modules if m in succeeded]) + output_modules = sorted([m for m in self.preset.output_modules if m in succeeded]) + internal_modules = sorted([m for m in self.preset.internal_modules if m in succeeded]) # Load scan modules self.verbose(f"Loading {len(modules):,} scan modules: {','.join(modules)}") @@ -504,7 +445,7 @@ async def load_modules(self): self._fail_setup(msg) if loaded_modules: self.info( - f"Loaded {len(loaded_modules):,}/{len(self._scan_modules):,} scan modules ({','.join(loaded_modules)})" + f"Loaded {len(loaded_modules):,}/{len(self.preset.scan_modules):,} scan modules ({','.join(loaded_modules)})" ) # Load internal modules @@ -516,7 +457,7 @@ async def load_modules(self): self._fail_setup(msg) if loaded_internal_modules: self.info( - f"Loaded {len(loaded_internal_modules):,}/{len(self._internal_modules):,} internal modules ({','.join(loaded_internal_modules)})" + f"Loaded {len(loaded_internal_modules):,}/{len(self.preset.internal_modules):,} internal modules ({','.join(loaded_internal_modules)})" ) # Load output modules @@ -528,7 +469,7 @@ async def load_modules(self): self._fail_setup(msg) if loaded_output_modules: self.info( - f"Loaded {len(loaded_output_modules):,}/{len(self._output_modules):,} output modules, ({','.join(loaded_output_modules)})" + f"Loaded {len(loaded_output_modules):,}/{len(self.preset.output_modules):,} output modules, ({','.join(loaded_output_modules)})" ) self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "_priority", 0))) @@ -705,6 +646,42 @@ def whitelisted(self, e): e = make_event(e, dummy=True) return e in self.whitelist + @property + def core(self): + return self.preset.core + + @property + def config(self): + return self.preset.core.config + + @property + def target(self): + return self.preset.target + + @property + def whitelist(self): + return self.preset.whitelist + + @property + def blacklist(self): + return self.preset.blacklist + + @property + def home(self): + return self.preset.scan_home + + @property + def name(self): + return self.preset.scan_name + + @property + def id(self): + return self.preset.scan_id + + @property + def helpers(self): + return self.preset.helpers + @property def word_cloud(self): return self.helpers.word_cloud @@ -929,7 +906,7 @@ def log_level(self): """ Return the current log level, e.g. logging.INFO """ - return CORE.log_level + return self.core.log_level @property def _log_handlers(self): @@ -952,26 +929,21 @@ def _start_log_handlers(self): # PRESET TODO: revisit scan logging # add log handlers for handler in self._log_handlers: - CORE.logger.add_log_handler(handler) + self.core.logger.add_log_handler(handler) # temporarily disable main ones for handler_name in ("file_main", "file_debug"): - handler = CORE.logger.log_handlers.get(handler_name, None) + handler = self.core.logger.log_handlers.get(handler_name, None) if handler is not None and handler not in self._log_handler_backup: self._log_handler_backup.append(handler) - CORE.logger.remove_log_handler(handler) + self.core.logger.remove_log_handler(handler) def _stop_log_handlers(self): # remove log handlers for handler in self._log_handlers: - CORE.logger.remove_log_handler(handler) + self.core.logger.remove_log_handler(handler) # restore main ones for handler in self._log_handler_backup: - CORE.logger.add_log_handler(handler) - - def _internal_modules(self): - for modname in CORE.module_loader.preloaded(type="internal"): - if self.config.get(modname, True): - yield modname + self.core.logger.add_log_handler(handler) def _fail_setup(self, msg): msg = str(msg) @@ -992,7 +964,7 @@ def _load_modules(self, modules): modules = [str(m) for m in modules] loaded_modules = {} failed = set() - for module_name, module_class in CORE.module_loader.load_modules(modules).items(): + for module_name, module_class in self.preset.module_loader.load_modules(modules).items(): if module_class: try: loaded_modules[module_name] = module_class(self) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 20e8bbf64..398bee661 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -62,7 +62,7 @@ class Target: - If you do not want to include child subdomains, use `strict_scope=True` """ - def __init__(self, scan, *targets, strict_scope=False, make_in_scope=False): + def __init__(self, preset, *targets, strict_scope=False, make_in_scope=False): """ Initialize a Target object. @@ -83,11 +83,12 @@ def __init__(self, scan, *targets, strict_scope=False, make_in_scope=False): - The strict_scope flag can be set to restrict scope calculation to only exactly-matching hosts and not their child subdomains. - Each target is processed and stored as an `Event` in the '_events' dictionary. """ - self.scan = scan + self.preset = preset + self.scan = self.preset.scan self.strict_scope = strict_scope self.make_in_scope = make_in_scope - self._dummy_module = TargetDummyModule(scan) + self._dummy_module = TargetDummyModule(self.scan) self._events = dict() if len(targets) > 0: log.verbose(f"Creating events from {len(targets):,} targets") @@ -179,7 +180,7 @@ def copy(self): Notes: - The `scan` object reference is kept intact in the copied Target object. """ - self_copy = self.__class__(self.scan, strict_scope=self.strict_scope) + self_copy = self.__class__(self.preset, strict_scope=self.strict_scope) self_copy._events = dict(self._events) return self_copy @@ -212,12 +213,12 @@ def get(self, host): if other.host: with suppress(KeyError, StopIteration): return next(iter(self._events[other.host])) - if self.scan.helpers.is_ip_type(other.host): - for n in self.scan.helpers.ip_network_parents(other.host, include_self=True): + if self.preset.helpers.is_ip_type(other.host): + for n in self.preset.helpers.ip_network_parents(other.host, include_self=True): with suppress(KeyError, StopIteration): return next(iter(self._events[n])) elif not self.strict_scope: - for h in self.scan.helpers.domain_parents(other.host): + for h in self.preset.helpers.domain_parents(other.host): with suppress(KeyError, StopIteration): return next(iter(self._events[h])) diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index 862ae5da3..e3bacc5ac 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -96,7 +96,7 @@ def update_individual_module_options(): update_md_files("BBOT MODULES", bbot_module_table) # BBOT output modules - bbot_output_module_table = module_loader.modules_table(mod_type="output") + bbot_output_module_table = CORE.module_loader.modules_table(mod_type="output") assert len(bbot_output_module_table.splitlines()) > 10 update_md_files("BBOT OUTPUT MODULES", bbot_output_module_table) From 22d0d169490f8f5b33475265bf37ba97bd8088c0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 6 Mar 2024 13:35:03 -0500 Subject: [PATCH 019/171] decoupling preset-->scan dependency --- bbot/core/helpers/cloud.py | 22 +++++----------------- bbot/core/helpers/helper.py | 21 --------------------- bbot/modules/internal/excavate.py | 9 ++++++++- bbot/modules/internal/speculate.py | 7 ++++++- bbot/scanner/scanner.py | 23 +++++++++++++++++++++++ 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/bbot/core/helpers/cloud.py b/bbot/core/helpers/cloud.py index 811ca070c..677efa093 100644 --- a/bbot/core/helpers/cloud.py +++ b/bbot/core/helpers/cloud.py @@ -10,11 +10,6 @@ class CloudHelper: def __init__(self, parent_helper): self.parent_helper = parent_helper self.providers = cloud_providers - self.dummy_modules = {} - for provider_name in self.providers.providers: - self.dummy_modules[provider_name] = self.parent_helper._make_dummy_module( - f"{provider_name}_cloud", _type="scan" - ) self._updated = False self._update_lock = asyncio.Lock() @@ -36,7 +31,7 @@ def excavate(self, event, s): if event_type == "STORAGE_BUCKET": self.emit_bucket(match, **kwargs) else: - self.emit_event(**kwargs) + yield kwargs def speculate(self, event): """ @@ -58,21 +53,14 @@ def speculate(self, event): if not event.data in found: found.add(event.data) if event_type == "STORAGE_BUCKET": - self.emit_bucket(match.groups(), **kwargs) + yield self.emit_bucket(match.groups(), **kwargs) else: - self.emit_event(**kwargs) + yield kwargs - def emit_bucket(self, match, **kwargs): + async def emit_bucket(self, match, **kwargs): bucket_name, bucket_domain = match kwargs["data"] = {"name": bucket_name, "url": f"https://{bucket_name}.{bucket_domain}"} - self.emit_event(**kwargs) - - def emit_event(self, *args, **kwargs): - provider_name = kwargs.pop("_provider") - dummy_module = self.dummy_modules[provider_name] - event = dummy_module.make_event(*args, **kwargs) - if event: - self.parent_helper.scan.manager.queue_event(event) + return kwargs async def tag_event(self, event): """ diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 9fa5eb1a5..58b37bf41 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -10,7 +10,6 @@ from .wordcloud import WordCloud from .interactsh import Interactsh from ...scanner.target import Target -from ...modules.base import BaseModule from .depsinstaller import DepsInstaller @@ -110,17 +109,6 @@ def scan(self): def in_tests(self): return os.environ.get("BBOT_TESTING", "") == "True" - def _make_dummy_module(self, name, _type="scan"): - """ - Construct a dummy module, for attachment to events - """ - try: - return self.dummy_modules[name] - except KeyError: - dummy = DummyModule(scan=self.scan, name=name, _type=_type) - self.dummy_modules[name] = dummy - return dummy - def __getattribute__(self, attr): """ Do not be afraid, the angel said. @@ -162,12 +150,3 @@ def __getattribute__(self, attr): except AttributeError: # then die raise AttributeError(f'Helper has no attribute "{attr}"') - - -class DummyModule(BaseModule): - _priority = 4 - - def __init__(self, *args, **kwargs): - self._name = kwargs.pop("name") - self._type = kwargs.pop("_type") - super().__init__(*args, **kwargs) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 24322d06b..0758ca2b4 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -387,8 +387,15 @@ async def handle_event(self, event): self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}") body = self.helpers.recursive_decode(event.data.get("body", "")) + # Cloud extractors - self.helpers.cloud.excavate(event, body) + for cloud_kwargs in self.helpers.cloud.excavate(event, body): + module = None + provider = kwargs.pop("_provider", "") + if provider: + module = self.scan._make_dummy_module(provider) + await self.emit_event(module=module, **kwargs) + await self.search( body, [ diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 9ae587200..399222bd8 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -131,7 +131,12 @@ async def handle_event(self, event): ) # storage buckets etc. - self.helpers.cloud.speculate(event) + for cloud_kwargs in self.helpers.cloud.speculate(event): + module = None + provider = kwargs.pop("_provider", "") + if provider: + module = self.scan._make_dummy_module(provider) + await self.emit_event(module=module, **kwargs) # ORG_STUB from TLD, SOCIAL, AZURE_TENANT org_stubs = set() diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index de345da62..a7fa95f9b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -136,6 +136,7 @@ def __init__( self.modules = OrderedDict({}) self._modules_loaded = False + self.dummy_modules = {} if dispatcher is None: self.dispatcher = Dispatcher() @@ -1013,3 +1014,25 @@ def _handle_exception(self, e, context="scan", finally_callback=None): log.trace(traceback.format_exc()) if callable(finally_callback): finally_callback(e) + + def _make_dummy_module(self, name, _type="scan"): + """ + Construct a dummy module, for attachment to events + """ + try: + return self.dummy_modules[name] + except KeyError: + dummy = DummyModule(scan=self, name=name, _type=_type) + self.dummy_modules[name] = dummy + return dummy + + +from bbot.modules.base import BaseModule + +class DummyModule(BaseModule): + _priority = 4 + + def __init__(self, *args, **kwargs): + self._name = kwargs.pop("name") + self._type = kwargs.pop("_type") + super().__init__(*args, **kwargs) From f81c197ef8874521695f40f9aab8f63ca31ef481 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 7 Mar 2024 09:36:02 -0500 Subject: [PATCH 020/171] steady work on presets --- bbot/cli.py | 38 +++++----- bbot/core/helpers/dns.py | 13 +--- bbot/core/helpers/web.py | 3 +- bbot/modules/internal/excavate.py | 4 +- bbot/modules/internal/speculate.py | 4 +- bbot/scanner/manager.py | 8 +- bbot/scanner/preset/args.py | 116 +++++++++++++---------------- bbot/scanner/preset/preset.py | 72 +++++++++++++----- bbot/scanner/scanner.py | 63 ++++++---------- bbot/scanner/target.py | 6 +- 10 files changed, 162 insertions(+), 165 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 10330e29b..6bdaf0ff7 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -20,16 +20,9 @@ from bbot.core import errors from bbot import __version__ -from bbot.modules import module_loader -from bbot.core.configurator.args import parser -from bbot.core.helpers.misc import smart_decode from bbot.core.helpers.logger import log_to_stderr log = logging.getLogger("bbot.cli") -log_level = get_log_level() - - -from . import config err = False scan_name = "" @@ -40,8 +33,25 @@ async def _main(): global scan_name from bbot.scanner import Scanner + from bbot.scanner.preset import Preset + + preset = Preset("www.example.com") + preset.parse_args() + + # --version + if preset.args.parsed.version: + log.stdout(__version__) + sys.exit(0) + return + + # --current-config + if preset.args.parsed.current_config: + log.stdout(f"{OmegaConf.to_yaml(CORE.config)}") + sys.exit(0) + return + + scan = Scanner(preset=preset) - scan = Scanner("www.example.com", _cli_execution=True) async for event in scan.async_start(): print(event) @@ -68,18 +78,6 @@ async def _main(): CORE.args.validate() options = CORE.args.parsed - # --version - if options.version: - log.stdout(__version__) - sys.exit(0) - return - - # --current-config - if options.current_config: - log.stdout(f"{OmegaConf.to_yaml(CORE.config)}") - sys.exit(0) - return - if options.agent_mode: from bbot.agent import Agent diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 6cbaf9f8e..463a1bb5a 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -551,10 +551,10 @@ async def resolve_event(self, event, minimal=False): if rdtype in ("A", "AAAA", "CNAME"): with contextlib.suppress(ValidationError): if self.parent_helper.is_ip(ip): - if self.parent_helper.scan.whitelisted(ip): + if self.parent_helper.preset.whitelisted(ip): event_whitelisted = True with contextlib.suppress(ValidationError): - if self.parent_helper.scan.blacklisted(ip): + if self.parent_helper.preset.blacklisted(ip): event_blacklisted = True if self.filter_bad_ptrs and rdtype in ("PTR") and self.parent_helper.is_ptr(t): @@ -1016,12 +1016,3 @@ def _parse_rdtype(self, t, default=None): def debug(self, *args, **kwargs): if self._debug: log.trace(*args, **kwargs) - - def _get_dummy_module(self, name): - try: - dummy_module = self._dummy_modules[name] - except KeyError: - dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") - dummy_module.suppress_dupes = False - self._dummy_modules[name] = dummy_module - return dummy_module diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index d754a53b7..87df73e6d 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -51,7 +51,6 @@ class BBOTAsyncClient(httpx.AsyncClient): def __init__(self, *args, **kwargs): self._preset = kwargs.pop("_preset") - self._bbot_scan = self._preset.scan web_requests_per_second = self._preset.config.get("web_requests_per_second", 100) self._rate_limiter = RateLimiter(web_requests_per_second, "Web") @@ -90,7 +89,7 @@ async def request(self, *args, **kwargs): def build_request(self, *args, **kwargs): request = super().build_request(*args, **kwargs) # add custom headers if the URL is in-scope - if self._bbot_scan.in_scope(str(request.url)): + if self._preset.in_scope(str(request.url)): for hk, hv in self._preset.config.get("http_headers", {}).items(): # don't clobber headers if hk not in request.headers: diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 0758ca2b4..1623dc700 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -391,10 +391,10 @@ async def handle_event(self, event): # Cloud extractors for cloud_kwargs in self.helpers.cloud.excavate(event, body): module = None - provider = kwargs.pop("_provider", "") + provider = cloud_kwargs.pop("_provider", "") if provider: module = self.scan._make_dummy_module(provider) - await self.emit_event(module=module, **kwargs) + await self.emit_event(module=module, **cloud_kwargs) await self.search( body, diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 399222bd8..e6476c63e 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -133,10 +133,10 @@ async def handle_event(self, event): # storage buckets etc. for cloud_kwargs in self.helpers.cloud.speculate(event): module = None - provider = kwargs.pop("_provider", "") + provider = cloud_kwargs.pop("_provider", "") if provider: module = self.scan._make_dummy_module(provider) - await self.emit_event(module=module, **kwargs) + await self.emit_event(module=module, **cloud_kwargs) # ORG_STUB from TLD, SOCIAL, AZURE_TENANT org_stubs = set() diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 96929e4fc..0fd95a0a0 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -72,6 +72,10 @@ async def init_events(self): await self.distribute_event(self.scan.root_event) sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) for event in sorted_events: + event._dummy = False + event.scan = self.scan + event.source = self.scan.root_event + event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") self.scan.verbose(f"Target: {event}") self.queue_event(event) await asyncio.sleep(0.1) @@ -277,7 +281,7 @@ async def _emit_event(self, event, **kwargs): and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") ): - source_module = self.scan.helpers._make_dummy_module("host", _type="internal") + source_module = self.scan._make_dummy_module("host", _type="internal") source_module._priority = 4 source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event) # only emit the event if it's not already in the parent chain @@ -301,7 +305,7 @@ async def _emit_event(self, event, **kwargs): dns_child_events = [] if dns_children: for rdtype, records in dns_children.items(): - module = self.scan.helpers.dns._get_dummy_module(rdtype) + module = self.scan._make_dummy_module_dns(rdtype) module._priority = 4 for record in records: try: diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index fa7e3071e..093f625d1 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -4,43 +4,15 @@ from pathlib import Path from omegaconf import OmegaConf -from ..helpers.logger import log_to_stderr -from ..helpers.misc import chain_lists, match_and_exit, is_file +from bbot.core.helpers.logger import log_to_stderr +from bbot.core.helpers.misc import chain_lists, match_and_exit, is_file -class BBOTArgumentParser(argparse.ArgumentParser): - """ - A subclass of argparse.ArgumentParser with several extra features: - - permissive parsing of modules/flags, allowing either space or comma-separated entries - - loading targets / config from files - - option to *not* exit on error - """ +class BBOTArgs: # module config options to exclude from validation exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$") - def parse_args(self, *args, **kwargs): - """ - Allow space or comma-separated entries for modules and targets - For targets, also allow input files containing additional targets - """ - ret = super().parse_args(*args, **kwargs) - # silent implies -y - if ret.silent: - ret.yes = True - ret.modules = chain_lists(ret.modules) - ret.exclude_modules = chain_lists(ret.exclude_modules) - ret.output_modules = chain_lists(ret.output_modules) - ret.targets = chain_lists(ret.targets, try_files=True, msg="Reading targets from file: {filename}") - ret.whitelist = chain_lists(ret.whitelist, try_files=True, msg="Reading whitelist from file: {filename}") - ret.blacklist = chain_lists(ret.blacklist, try_files=True, msg="Reading blacklist from file: {filename}") - ret.flags = chain_lists(ret.flags) - ret.exclude_flags = chain_lists(ret.exclude_flags) - ret.require_flags = chain_lists(ret.require_flags) - return ret - - -class BBOTArgs: scan_examples = [ ( "Subdomains", @@ -94,10 +66,9 @@ class BBOTArgs: def __init__(self, preset): self.preset = preset - self._parsed = None - self._cli_config = None + self._config = None - # PRESET TODO: revisit this + # validate module choices self._module_choices = sorted(set(self.preset.module_loader.preloaded(type="scan"))) self._output_module_choices = sorted(set(self.preset.module_loader.preloaded(type="output"))) self._flag_choices = set() @@ -106,20 +77,18 @@ def __init__(self, preset): self._flag_choices = sorted(self._flag_choices) self.parser = self.create_parser() + self._parsed = None @property def parsed(self): - """ - Returns the parsed BBOT Argument Parser. - """ if self._parsed is None: self._parsed = self.parser.parse_args() return self._parsed @property - def cli_config(self): - if self._cli_config is None: - self._cli_config = OmegaConf.create({}) + def config(self): + if self._config is None: + self._config = OmegaConf.create({}) if self.parsed.config: for c in self.parsed.config: config_file = Path(c).resolve() @@ -136,8 +105,8 @@ def cli_config(self): except Exception as e: log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") sys.exit(2) - self._cli_config = OmegaConf.merge(self._cli_config, cli_config) - return self._cli_config + self._config = OmegaConf.merge(self._config, cli_config) + return self._config def create_parser(self, *args, **kwargs): kwargs.update( @@ -145,7 +114,7 @@ def create_parser(self, *args, **kwargs): description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=self.epilog ) ) - p = BBOTArgumentParser(*args, **kwargs) + p = argparse.ArgumentParser(*args, **kwargs) p.add_argument("--help-all", action="store_true", help="Display full help including module config options") target = p.add_argument_group(title="Target") target.add_argument( @@ -254,28 +223,29 @@ def create_parser(self, *args, **kwargs): misc.add_argument("--version", action="store_true", help="show BBOT version and exit") return p - def validate(self): - """ - Validate command line arguments - - Validate modules/flags to make sure they exist - - If --config was specified, check the config options to make sure they exist - """ - # modules - for m in self.parsed.modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in self.parsed.exclude_modules: - if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") - for m in self.parsed.output_modules: - if m not in self._output_module_choices: - match_and_exit(m, self._output_module_choices, msg="output module") - # flags - for f in set(self.parsed.flags + self.parsed.require_flags + self.parsed.exclude_flags): - if f not in self._flag_choices: - match_and_exit(f, self._flag_choices, msg="flag") + def sanitize_args(self): + # silent implies -y + if self.parsed.silent: + self.parsed.yes = True + # chain_lists allows either comma-separated or space-separated lists + self.parsed.modules = chain_lists(self.parsed.modules) + self.parsed.exclude_modules = chain_lists(self.parsed.exclude_modules) + self.parsed.output_modules = chain_lists(self.parsed.output_modules) + self.parsed.targets = chain_lists( + self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}" + ) + self.parsed.whitelist = chain_lists( + self.parsed.whitelist, try_files=True, msg="Reading whitelist from file: {filename}" + ) + self.parsed.blacklist = chain_lists( + self.parsed.blacklist, try_files=True, msg="Reading blacklist from file: {filename}" + ) + self.parsed.flags = chain_lists(self.parsed.flags) + self.parsed.exclude_flags = chain_lists(self.parsed.exclude_flags) + self.parsed.require_flags = chain_lists(self.parsed.require_flags) - # config options + def validate(self): + # validate config options sentinel = object() conf = [a for a in self.parsed.config if not is_file(a)] all_options = None @@ -295,3 +265,21 @@ def validate(self): global_options = set(self.preset.core.default_config.keys()) - {"modules", "output_modules"} all_options = global_options.union(modules_options) match_and_exit(c, all_options, msg="module option") + + # PRESET TODO: if custom module dir, pull in new module choices + + # validate modules + for m in self.parser.modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in self.parser.exclude_modules: + if m not in self._module_choices: + match_and_exit(m, self._module_choices, msg="module") + for m in self.parser.output_modules: + if m not in self._output_module_choices: + match_and_exit(m, self._output_module_choices, msg="output module") + + # validate flags + for f in set(self.parser.flags + self.parser.require_flags + self.parser.exclude_flags): + if f not in self._flag_choices: + match_and_exit(f, self._flag_choices, msg="flag") diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 3da0aef9e..d9874ae18 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -2,6 +2,8 @@ from omegaconf import OmegaConf from bbot.core import CORE +from bbot.core.event.base import make_event +from bbot.core.errors import ValidationError from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.names_generator import random_name @@ -21,7 +23,6 @@ class Preset: def __init__( self, *targets, - scan=None, whitelist=None, blacklist=None, scan_id=None, @@ -34,8 +35,6 @@ def __init__( strict_scope=False, _cli_execution=False, ): - self._scan = scan - self._args = None self._environ = None self._module_loader = None @@ -48,6 +47,7 @@ def __init__( # merge any custom configs self.core.merge_custom(config) + # modules if modules is None: modules = [] if output_modules is None: @@ -74,6 +74,7 @@ def __init__( self.helpers = ConfigAwareHelper(preset=self) + # scan ID if scan_id is not None: self.scan_id = str(scan_id) else: @@ -103,7 +104,6 @@ def __init__( else: self.scan_home = self.helpers.bbot_home / "scans" / self.scan_name - def set_scope(self, targets, whitelist, blacklist, strict_scope=False): self.strict_scope = strict_scope # target / whitelist / blacklist @@ -118,21 +118,27 @@ def set_scope(self, targets, whitelist, blacklist, strict_scope=False): blacklist = [] self.blacklist = Target(self, *blacklist) - def process_cli_args(self): - pass + def parse_args(self): + + from .args import BBOTArgs + + self._args = BBOTArgs(self) + + # bring in presets + # self.merge(self.args.presets) + + # bring in config + self.core.merge_custom(self.args.config) + + # bring in misc cli arguments + + # validate config / modules / flags + # self.args.validate() @property def config(self): return self.core.config - @property - def scan(self): - if self._scan is None: - from bbot.scanner import Scanner - - self._scan = Scanner() - return self._scan - @property def module_loader(self): # module loader depends on environment to be set up @@ -174,8 +180,38 @@ def environ(self): @property def args(self): - if self._args is None: - from .args import BBOTArgs - - self._args = BBOTArgs(self) return self._args + + def in_scope(self, e): + """ + Check whether a hostname, url, IP, etc. is in scope. + Accepts either events or string data. + + Checks whitelist and blacklist. + If `e` is an event and its scope distance is zero, it will be considered in-scope. + + Examples: + Check if a URL is in scope: + >>> scan.in_scope("http://www.evilcorp.com") + True + """ + try: + e = make_event(e, dummy=True) + except ValidationError: + return False + in_scope = e.scope_distance == 0 or self.whitelisted(e) + return in_scope and not self.blacklisted(e) + + def blacklisted(self, e): + """ + Check whether a hostname, url, IP, etc. is blacklisted. + """ + e = make_event(e, dummy=True) + return e in self.blacklist + + def whitelisted(self, e): + """ + Check whether a hostname, url, IP, etc. is whitelisted. + """ + e = make_event(e, dummy=True) + return e in self.whitelist diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a7fa95f9b..3b02a6532 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -19,8 +19,8 @@ from .manager import ScanManager from .dispatcher import Dispatcher from bbot.core.event import make_event +from bbot.core.errors import BBOTError, ScanError from bbot.core.helpers.async_helpers import async_to_sync_gen -from bbot.core.errors import BBOTError, ScanError, ValidationError log = logging.getLogger("bbot.scanner") @@ -99,10 +99,7 @@ class Scanner: def __init__( self, - *targets, - whitelist=None, - blacklist=None, - strict_scope=False, + *args, dispatcher=None, force_start=False, **preset_kwargs, @@ -125,8 +122,11 @@ def __init__( force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. """ - self.preset = Preset(scan=self, **preset_kwargs) - self.preset.set_scope(targets, whitelist, blacklist, strict_scope=strict_scope) + preset = preset_kwargs.pop("preset", None) + if preset is not None: + self.preset = preset + else: + self.preset = Preset(*args, **preset_kwargs) self.force_start = force_start self._status = "NOT_STARTED" @@ -613,39 +613,14 @@ async def _cleanup(self): self.home.rmdir() self.helpers.clean_old_scans() - def in_scope(self, e): - """ - Check whether a hostname, url, IP, etc. is in scope. - Accepts either events or string data. - - Checks whitelist and blacklist. - If `e` is an event and its scope distance is zero, it will be considered in-scope. + def in_scope(self, *args, **kwargs): + return self.preset.in_scope(*args, **kwargs) - Examples: - Check if a URL is in scope: - >>> scan.in_scope("http://www.evilcorp.com") - True - """ - try: - e = make_event(e, dummy=True) - except ValidationError: - return False - in_scope = e.scope_distance == 0 or self.whitelisted(e) - return in_scope and not self.blacklisted(e) + def whitelisted(self, *args, **kwargs): + return self.preset.whitelisted(*args, **kwargs) - def blacklisted(self, e): - """ - Check whether a hostname, url, IP, etc. is blacklisted. - """ - e = make_event(e, dummy=True) - return e in self.blacklist - - def whitelisted(self, e): - """ - Check whether a hostname, url, IP, etc. is whitelisted. - """ - e = make_event(e, dummy=True) - return e in self.whitelist + def blacklisted(self, *args, **kwargs): + return self.preset.blacklisted(*args, **kwargs) @property def core(self): @@ -759,7 +734,7 @@ def root_event(self): root_event.scope_distance = 0 root_event._resolved.set() root_event.source = root_event - root_event.module = self.helpers._make_dummy_module(name="TARGET", _type="TARGET") + root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") return root_event def run_in_executor(self, callback, *args, **kwargs): @@ -1026,9 +1001,19 @@ def _make_dummy_module(self, name, _type="scan"): self.dummy_modules[name] = dummy return dummy + def _make_dummy_module_dns(self, name): + try: + dummy_module = self.dummy_modules[name] + except KeyError: + dummy_module = self._make_dummy_module(name=name, _type="DNS") + dummy_module.suppress_dupes = False + self.dummy_modules[name] = dummy_module + return dummy_module + from bbot.modules.base import BaseModule + class DummyModule(BaseModule): _priority = 4 diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 398bee661..eb52206e7 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -84,11 +84,9 @@ def __init__(self, preset, *targets, strict_scope=False, make_in_scope=False): - Each target is processed and stored as an `Event` in the '_events' dictionary. """ self.preset = preset - self.scan = self.preset.scan self.strict_scope = strict_scope self.make_in_scope = make_in_scope - self._dummy_module = TargetDummyModule(self.scan) self._events = dict() if len(targets) > 0: log.verbose(f"Creating events from {len(targets):,} targets") @@ -125,9 +123,7 @@ def add_target(self, t): if is_event(t): event = t else: - event = self.scan.make_event( - t, source=self.scan.root_event, module=self._dummy_module, tags=["target"] - ) + event = make_event(t, tags=["target"], dummy=True) if self.make_in_scope: event.scope_distance = 0 try: From a7dfb0fb73dd644d9c07533f94d03df56a3cba01 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 8 Mar 2024 06:26:58 -0500 Subject: [PATCH 021/171] steady work on presets --- bbot/cli.py | 16 ++++--- bbot/core/helpers/helper.py | 2 +- bbot/scanner/preset/args.py | 61 +++++++++++++----------- bbot/scanner/preset/preset.py | 89 ++++++++++++++++++++++++++++++----- bbot/scanner/scanner.py | 5 ++ 5 files changed, 125 insertions(+), 48 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 6bdaf0ff7..98e25aaf5 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -9,8 +9,6 @@ from omegaconf import OmegaConf from contextlib import suppress -# from aioconsole import stream - # fix tee buffering sys.stdout.reconfigure(line_buffering=True) @@ -35,7 +33,7 @@ async def _main(): from bbot.scanner import Scanner from bbot.scanner.preset import Preset - preset = Preset("www.example.com") + preset = Preset() preset.parse_args() # --version @@ -44,9 +42,15 @@ async def _main(): sys.exit(0) return - # --current-config - if preset.args.parsed.current_config: - log.stdout(f"{OmegaConf.to_yaml(CORE.config)}") + # --current-preset + if preset.args.parsed.current_preset: + log.stdout(preset.to_yaml()) + sys.exit(0) + return + + # --current-preset-full + if preset.args.parsed.current_preset_full: + log.stdout(preset.to_yaml(full_config=True)) sys.exit(0) return diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 58b37bf41..d416ac768 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -52,7 +52,7 @@ class ConfigAwareHelper: def __init__(self, preset): self.preset = preset - self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() + self.bbot_home = self.preset.bbot_home self.cache_dir = self.bbot_home / "cache" self.temp_dir = self.bbot_home / "temp" self.tools_dir = self.bbot_home / "tools" diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 093f625d1..e8ed87594 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -85,28 +85,26 @@ def parsed(self): self._parsed = self.parser.parse_args() return self._parsed - @property - def config(self): - if self._config is None: - self._config = OmegaConf.create({}) - if self.parsed.config: - for c in self.parsed.config: - config_file = Path(c).resolve() - if config_file.is_file(): - try: - cli_config = OmegaConf.load(str(config_file)) - log_to_stderr(f"Loaded custom config from {config_file}") - except Exception as e: - log_to_stderr(f"Error parsing custom config at {config_file}: {e}", level="ERROR") - sys.exit(2) - else: - try: - cli_config = OmegaConf.from_cli(cli_config) - except Exception as e: - log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") - sys.exit(2) - self._config = OmegaConf.merge(self._config, cli_config) - return self._config + def preset_from_args(self, args=None): + if args is None: + args = self.parsed.preset + args_preset = self.preset.__class__() + for preset in args: + if Path(preset).is_file(): + try: + custom_preset = self.preset.from_yaml(preset) + except Exception as e: + log_to_stderr(f"Error parsing custom config at {config_file}: {e}", level="ERROR") + sys.exit(2) + args_preset.merge(custom_preset) + else: + try: + cli_config = OmegaConf.from_cli([preset]) + except Exception as e: + log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") + sys.exit(2) + args_preset.core.merge_custom(cli_config) + return args_preset def create_parser(self, *args, **kwargs): kwargs.update( @@ -188,10 +186,10 @@ def create_parser(self, *args, **kwargs): metavar="DIR", ) scan.add_argument( - "-c", - "--config", + "-p", "--preset", + "-c", "--config", nargs="*", - help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", + help="Custom preset file(s), or config options in key=value format: 'modules.shodan.api_key=1234'", metavar="CONFIG", default=[], ) @@ -202,9 +200,14 @@ def create_parser(self, *args, **kwargs): scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") scan.add_argument( - "--current-config", + "--current-preset", + action="store_true", + help="Show the current preset in YAML format", + ) + scan.add_argument( + "--current-preset-full", action="store_true", - help="Show current config in YAML format", + help="Show the current preset in its full form, including defaults", ) deps = p.add_argument_group( title="Module dependencies", description="Control how modules install their dependencies" @@ -247,13 +250,14 @@ def sanitize_args(self): def validate(self): # validate config options sentinel = object() - conf = [a for a in self.parsed.config if not is_file(a)] + conf = [a for a in self.parsed.preset if not is_file(a)] all_options = None for c in conf: c = c.split("=")[0].strip() v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel) # if option isn't in the default config if v is sentinel: + # skip if it's excluded from validation if self.exclude_from_validation.match(c): continue if all_options is None: @@ -264,6 +268,7 @@ def validate(self): modules_options.update(set(o[0] for o in module_options)) global_options = set(self.preset.core.default_config.keys()) - {"modules", "output_modules"} all_options = global_options.union(modules_options) + # otherwise, ensure it exists as a module option match_and_exit(c, all_options, msg="module option") # PRESET TODO: if custom module dir, pull in new module choices diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index d9874ae18..50cb3df12 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,3 +1,4 @@ +import yaml from pathlib import Path from omegaconf import OmegaConf @@ -25,7 +26,6 @@ def __init__( *targets, whitelist=None, blacklist=None, - scan_id=None, scan_name=None, modules=None, output_modules=None, @@ -33,12 +33,11 @@ def __init__( config=None, dispatcher=None, strict_scope=False, - _cli_execution=False, + helper=None, ): self._args = None self._environ = None self._module_loader = None - self._cli_execution = _cli_execution # bbot core config self.core = CORE @@ -70,22 +69,18 @@ def __init__( self.module_dirs = sorted(set(self.module_dirs)) # config-aware helper - from bbot.core.helpers.helper import ConfigAwareHelper + if helper is not None: + self._helpers = helper - self.helpers = ConfigAwareHelper(preset=self) - - # scan ID - if scan_id is not None: - self.scan_id = str(scan_id) - else: - self.scan_id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" + self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() # scan name + self._custom_scan_name = False if scan_name is None: tries = 0 while 1: if tries > 5: - self.scan_name = f"{self.helpers.rand_string(4)}_{self.helpers.rand_string(4)}" + self.scan_name = f"{rand_string(4)}_{rand_string(4)}" break self.scan_name = random_name() if output_dir is not None: @@ -97,12 +92,17 @@ def __init__( tries += 1 else: self.scan_name = str(scan_name) + self._custom_scan_name = True # scan output dir + self._custom_output_dir = False if output_dir is not None: self.scan_home = Path(output_dir).resolve() / self.scan_name + self._custom_output_dir = False else: self.scan_home = self.helpers.bbot_home / "scans" / self.scan_name + if self._custom_scan_name: + self._custom_output_dir = True self.strict_scope = strict_scope @@ -118,17 +118,36 @@ def __init__( blacklist = [] self.blacklist = Target(self, *blacklist) + def merge(self, other): + # module dirs + self.module_dirs = sorted(set(self.module_dirs).union(set(other.module_dirs))) + # scan name + if other.scan_name and other._custom_scan_name: + self.scan_name = other.scan_name + # scan output dir + if other.scan_home and other._custom_output_dir: + self.scan_home = other.scan_home + # merge target / whitelist / blacklist + self.target.add_target(other.target) + self.whitelist.add_target(other.whitelist) + self.blacklist.add_target(other.blacklist) + # scope + self.strict_scope = self.strict_scope or other.strict_scope + # config + self.core.merge_custom(other.core.custom_config) + def parse_args(self): from .args import BBOTArgs self._args = BBOTArgs(self) + self.merge(self.args.preset_from_args()) # bring in presets # self.merge(self.args.presets) # bring in config - self.core.merge_custom(self.args.config) + # self.core.merge_custom(self.args.config) # bring in misc cli arguments @@ -139,6 +158,14 @@ def parse_args(self): def config(self): return self.core.config + @property + def helpers(self): + if self._helpers is None: + from bbot.core.helpers.helper import ConfigAwareHelper + + self._helpers = ConfigAwareHelper(preset=self) + return self._helpers + @property def module_loader(self): # module loader depends on environment to be set up @@ -215,3 +242,39 @@ def whitelisted(self, e): """ e = make_event(e, dummy=True) return e in self.whitelist + + @classmethod + def from_yaml(cls, filename): + preset_dict = OmegaConf.load(filename) + new_preset = cls( + *preset_dict.get("targets", []), + whitelist=preset_dict.get("whitelist", []), + blacklist=preset_dict.get("blacklist", []), + scan_name=preset_dict.get("scan_name", None), + modules=preset_dict.get("modules", []), + output_modules=preset_dict.get("output_modules", []), + output_dir=preset_dict.get("output_dir", None), + config=preset_dict.get("config", None), + strict_scope=preset_dict.get("strict_scope", False), + ) + return new_preset + + def to_yaml(self, full_config=False): + if full_config: + config = self.core.config + else: + config = self.core.custom_config + target = sorted(str(t.data) for t in self.target) + whitelist = sorted(str(t.data) for t in self.whitelist) + blacklist = sorted(str(t.data) for t in self.blacklist) + preset_dict = { + "target": target, + "config": OmegaConf.to_container(config), + } + if whitelist and whitelist != target: + preset_dict["whitelist"] = whitelist + if blacklist: + preset_dict["blacklist"] = blacklist + if self.strict_scope: + preset_dict["strict_scope"] = True + return yaml.dump(preset_dict) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 3b02a6532..297d4e2dc 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -100,6 +100,7 @@ class Scanner: def __init__( self, *args, + scan_id=None, dispatcher=None, force_start=False, **preset_kwargs, @@ -121,6 +122,10 @@ def __init__( strict_scope (bool, optional): If True, only targets explicitly in whitelist are scanned. Defaults to False. force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. """ + if scan_id is not None: + self.scan_id = str(scan_id) + else: + self.scan_id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" preset = preset_kwargs.pop("preset", None) if preset is not None: From 02acd94d67c26821b8b8aa79ccc01d2aa6ae09ad Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 8 Mar 2024 16:43:16 -0500 Subject: [PATCH 022/171] integrating presets with command line args --- bbot/cli.py | 1 - bbot/core/helpers/depsinstaller/installer.py | 2 +- bbot/scanner/preset/args.py | 30 ++++--- bbot/scanner/preset/preset.py | 83 ++++++-------------- bbot/scanner/scanner.py | 45 +++++++---- bbot/scanner/target.py | 37 +++++---- 6 files changed, 94 insertions(+), 104 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index da3619160..d6139730e 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -6,7 +6,6 @@ import asyncio import logging import traceback -from omegaconf import OmegaConf from contextlib import suppress # fix tee buffering diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index fa32edf8b..d97fb577d 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -82,7 +82,7 @@ async def install(self, *modules): success = self.setup_status.get(module_hash, None) dependencies = list(chain(*preloaded["deps"].values())) if len(dependencies) <= 0: - log.debug(f'No setup to do for module "{m}"') + log.debug(f'No dependency work to do for module "{m}"') succeeded.append(m) continue else: diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index e8ed87594..4e02c0e75 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -83,23 +83,31 @@ def __init__(self, preset): def parsed(self): if self._parsed is None: self._parsed = self.parser.parse_args() + self.sanitize_args() return self._parsed - def preset_from_args(self, args=None): - if args is None: - args = self.parsed.preset + def preset_from_args(self): args_preset = self.preset.__class__() - for preset in args: - if Path(preset).is_file(): + # scope + args_preset.target.add_target(self.parsed.targets) + args_preset.whitelist.add_target(self.parsed.whitelist) + args_preset.blacklist.add_target(self.parsed.blacklist) + args_preset.strict_scope = self.parsed.strict_scope + # modules + args_preset.scan_modules = self.parsed.modules + args_preset.output_modules = self.parsed.output_modules + # additional custom presets / config options + for preset_param in self.parsed.preset: + if Path(preset_param).is_file(): try: - custom_preset = self.preset.from_yaml(preset) + custom_preset = self.preset.from_yaml(preset_param) except Exception as e: - log_to_stderr(f"Error parsing custom config at {config_file}: {e}", level="ERROR") + log_to_stderr(f"Error parsing custom config at {preset_param}: {e}", level="ERROR") sys.exit(2) args_preset.merge(custom_preset) else: try: - cli_config = OmegaConf.from_cli([preset]) + cli_config = OmegaConf.from_cli([preset_param]) except Exception as e: log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") sys.exit(2) @@ -186,8 +194,10 @@ def create_parser(self, *args, **kwargs): metavar="DIR", ) scan.add_argument( - "-p", "--preset", - "-c", "--config", + "-p", + "--preset", + "-c", + "--config", nargs="*", help="Custom preset file(s), or config options in key=value format: 'modules.shodan.api_key=1234'", metavar="CONFIG", diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 50cb3df12..015e7c07a 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -5,19 +5,12 @@ from bbot.core import CORE from bbot.core.event.base import make_event from bbot.core.errors import ValidationError -from bbot.core.helpers.misc import sha1, rand_string -from bbot.core.helpers.names_generator import random_name bbot_code_dir = Path(__file__).parent.parent.parent class Preset: - """ - CORE should not handle arguments - which means it shouldn't need access to the module loader - - """ default_module_dir = bbot_code_dir / "modules" @@ -26,17 +19,14 @@ def __init__( *targets, whitelist=None, blacklist=None, - scan_name=None, modules=None, output_modules=None, - output_dir=None, config=None, - dispatcher=None, strict_scope=False, - helper=None, ): self._args = None self._environ = None + self._helpers = None self._module_loader = None # bbot core config @@ -68,42 +58,6 @@ def __init__( self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] self.module_dirs = sorted(set(self.module_dirs)) - # config-aware helper - if helper is not None: - self._helpers = helper - - self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() - - # scan name - self._custom_scan_name = False - if scan_name is None: - tries = 0 - while 1: - if tries > 5: - self.scan_name = f"{rand_string(4)}_{rand_string(4)}" - break - self.scan_name = random_name() - if output_dir is not None: - home_path = Path(output_dir).resolve() / self.scan_name - else: - home_path = self.helpers.bbot_home / "scans" / self.scan_name - if not home_path.exists(): - break - tries += 1 - else: - self.scan_name = str(scan_name) - self._custom_scan_name = True - - # scan output dir - self._custom_output_dir = False - if output_dir is not None: - self.scan_home = Path(output_dir).resolve() / self.scan_name - self._custom_output_dir = False - else: - self.scan_home = self.helpers.bbot_home / "scans" / self.scan_name - if self._custom_scan_name: - self._custom_output_dir = True - self.strict_scope = strict_scope # target / whitelist / blacklist @@ -118,15 +72,19 @@ def __init__( blacklist = [] self.blacklist = Target(self, *blacklist) + self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() + def merge(self, other): # module dirs - self.module_dirs = sorted(set(self.module_dirs).union(set(other.module_dirs))) - # scan name - if other.scan_name and other._custom_scan_name: - self.scan_name = other.scan_name - # scan output dir - if other.scan_home and other._custom_output_dir: - self.scan_home = other.scan_home + current_module_dirs = set(self.module_dirs) + other_module_dirs = set(other.module_dirs) + combined_module_dirs = current_module_dirs.union(other_module_dirs) + if combined_module_dirs != current_module_dirs: + self.module_dirs = combined_module_dirs + # TODO: refresh module dirs + # modules + self.scan_modules = sorted(set(self.scan_modules).union(set(other.scan_modules))) + self.output_modules = sorted(set(self.output_modules).union(set(other.output_modules))) # merge target / whitelist / blacklist self.target.add_target(other.target) self.whitelist.add_target(other.whitelist) @@ -244,22 +202,23 @@ def whitelisted(self, e): return e in self.whitelist @classmethod - def from_yaml(cls, filename): - preset_dict = OmegaConf.load(filename) + def from_yaml(cls, yaml_preset): + if Path(yaml_preset).is_file(): + preset_dict = OmegaConf.load(yaml_preset) + else: + preset_dict = OmegaConf.create(yaml_preset) new_preset = cls( *preset_dict.get("targets", []), whitelist=preset_dict.get("whitelist", []), blacklist=preset_dict.get("blacklist", []), - scan_name=preset_dict.get("scan_name", None), - modules=preset_dict.get("modules", []), + modules=preset_dict.get("scan_modules", []), output_modules=preset_dict.get("output_modules", []), - output_dir=preset_dict.get("output_dir", None), config=preset_dict.get("config", None), strict_scope=preset_dict.get("strict_scope", False), ) return new_preset - def to_yaml(self, full_config=False): + def to_yaml(self, full_config=False, sort_keys=False): if full_config: config = self.core.config else: @@ -270,6 +229,8 @@ def to_yaml(self, full_config=False): preset_dict = { "target": target, "config": OmegaConf.to_container(config), + "modules": self.scan_modules, + "output_modules": self.output_modules, } if whitelist and whitelist != target: preset_dict["whitelist"] = whitelist @@ -277,4 +238,4 @@ def to_yaml(self, full_config=False): preset_dict["blacklist"] = blacklist if self.strict_scope: preset_dict["strict_scope"] = True - return yaml.dump(preset_dict) + return yaml.dump(preset_dict, sort_keys=sort_keys) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 297d4e2dc..fcb961625 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -4,6 +4,7 @@ import logging import traceback import contextlib +from pathlib import Path from sys import exc_info import multiprocessing as mp from datetime import datetime @@ -20,6 +21,8 @@ from .dispatcher import Dispatcher from bbot.core.event import make_event from bbot.core.errors import BBOTError, ScanError +from bbot.core.helpers.misc import sha1, rand_string +from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen log = logging.getLogger("bbot.scanner") @@ -101,6 +104,8 @@ def __init__( self, *args, scan_id=None, + scan_name=None, + output_dir=None, dispatcher=None, force_start=False, **preset_kwargs, @@ -123,9 +128,9 @@ def __init__( force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. """ if scan_id is not None: - self.scan_id = str(scan_id) + self.id = str(id) else: - self.scan_id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" + self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" preset = preset_kwargs.pop("preset", None) if preset is not None: @@ -133,6 +138,30 @@ def __init__( else: self.preset = Preset(*args, **preset_kwargs) + # scan name + if scan_name is None: + tries = 0 + while 1: + if tries > 5: + self.name = f"{rand_string(4)}_{rand_string(4)}" + break + self.name = random_name() + if output_dir is not None: + home_path = Path(output_dir).resolve() / self.name + else: + home_path = self.preset.bbot_home / "scans" / self.name + if not home_path.exists(): + break + tries += 1 + else: + self.name = str(scan_name) + + # scan output dir + if output_dir is not None: + self.home = Path(output_dir).resolve() / self.name + else: + self.home = self.preset.bbot_home / "scans" / self.name + self.force_start = force_start self._status = "NOT_STARTED" self._status_code = 0 @@ -647,18 +676,6 @@ def whitelist(self): def blacklist(self): return self.preset.blacklist - @property - def home(self): - return self.preset.scan_home - - @property - def name(self): - return self.preset.scan_name - - @property - def id(self): - return self.preset.scan_id - @property def helpers(self): return self.preset.helpers diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index eb52206e7..dddd78ff8 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -113,25 +113,28 @@ def add_target(self, t): - If `t` is an event, it is directly added to `_events`. - If `make_in_scope` is True, the scope distance of the event is set to 0. """ - if type(t) == self.__class__: - for k, v in t._events.items(): + if not isinstance(t, (list, tuple, set)): + t = [t] + for single_target in t: + if type(single_target) == self.__class__: + for k, v in single_target._events.items(): + try: + self._events[k].update(v) + except KeyError: + self._events[k] = set(single_target._events[k]) + else: + if is_event(single_target): + event = single_target + else: + event = make_event(single_target, tags=["target"], dummy=True) + if self.make_in_scope: + event.scope_distance = 0 try: - self._events[k].update(v) + self._events[event.host].add(event) except KeyError: - self._events[k] = set(t._events[k]) - else: - if is_event(t): - event = t - else: - event = make_event(t, tags=["target"], dummy=True) - if self.make_in_scope: - event.scope_distance = 0 - try: - self._events[event.host].add(event) - except KeyError: - self._events[event.host] = { - event, - } + self._events[event.host] = { + event, + } @property def events(self): From e51d5643c6ab48e8a48dc7d1cf11d1fa4e2a777d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 11 Mar 2024 15:37:12 -0400 Subject: [PATCH 023/171] steady work on presets --- bbot/cli.py | 346 +------------------------- bbot/core/core.py | 7 +- bbot/scanner/preset/args.py | 14 ++ bbot/scanner/preset/preset.py | 147 +++++++++-- bbot/scanner/target.py | 7 +- bbot/test/test_step_1/test_presets.py | 28 +-- 6 files changed, 172 insertions(+), 377 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index d6139730e..838a64280 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,21 +1,15 @@ #!/usr/bin/env python3 -import os -import re import sys import asyncio import logging import traceback -from contextlib import suppress # fix tee buffering sys.stdout.reconfigure(line_buffering=True) from bbot.core import CORE -CORE.logger.set_log_level("DEBUG") - -from bbot.core import errors from bbot import __version__ from bbot.core.helpers.logger import log_to_stderr @@ -26,13 +20,19 @@ async def _main(): - global err - global scan_name - from bbot.scanner import Scanner from bbot.scanner.preset import Preset + # start by creating a default scan preset preset = Preset() + + # print help if no arguments + if len(sys.argv) == 1: + preset.args.parser.print_help() + sys.exit(1) + return + + # parse command line arguments and merge into preset preset.parse_args() # --version @@ -55,333 +55,7 @@ async def _main(): scan = Scanner(preset=preset) - async for event in scan.async_start(): - print(event) - - # log.hugesuccess(CORE.default_config) - # log.hugeinfo(CORE.custom_config) - # log.hugewarning(CORE.module_loader.configs()) - # log.hugesuccess(CORE.default_config) - # print(CORE.module_loader) - - # 1) load core (environment variables, modules) - # 2) instantiate god preset - # 3) pull in command line arguments (this adds targets, configs, presets, etc.) - # 4) pass preset to scan - - return - - try: - - if len(sys.argv) == 1: - CORE.args.parser.print_help() - sys.exit(1) - - # command-line options are in bbot/core/config/args.py - CORE.args.validate() - options = CORE.args.parsed - - if options.agent_mode: - from bbot.agent import Agent - - agent = Agent(CORE.config) - success = agent.setup() - if success: - await agent.start() - - else: - from bbot.scanner import Scanner - - try: - output_modules = set(options.output_modules) - module_filtering = False - if (options.list_modules or options.help_all) and not any([options.flags, options.modules]): - module_filtering = True - modules = set(CORE.module_loader.preloaded(type="scan")) - else: - modules = set(options.modules) - # enable modules by flags - for m, c in CORE.module_loader.preloaded().items(): - module_type = c.get("type", "scan") - if m not in modules: - flags = c.get("flags", []) - if "deadly" in flags: - continue - for f in options.flags: - if f in flags: - log.verbose(f'Enabling {m} because it has flag "{f}"') - if module_type == "output": - output_modules.add(m) - else: - modules.add(m) - - default_output_modules = ["human", "json", "csv"] - - # Make a list of the modules which can be output to the console - consoleable_output_modules = [ - k for k, v in CORE.module_loader.preloaded(type="output").items() if "console" in v["config"] - ] - - # if none of the output modules provided on the command line are consoleable, don't turn off the defaults. Instead, just add the one specified to the defaults. - if not any(o in consoleable_output_modules for o in output_modules): - output_modules.update(default_output_modules) - - scanner = Scanner( - *options.targets, - modules=list(modules), - output_modules=list(output_modules), - output_dir=options.output_dir, - config=CORE.config, - name=options.name, - whitelist=options.whitelist, - blacklist=options.blacklist, - strict_scope=options.strict_scope, - force_start=options.force, - ) - - if options.install_all_deps: - all_modules = list(CORE.module_loader.preloaded()) - scanner.helpers.depsinstaller.force_deps = True - succeeded, failed = await scanner.helpers.depsinstaller.install(*all_modules) - log.info("Finished installing module dependencies") - return False if failed else True - - scan_name = str(scanner.name) - - # enable modules by dependency - # this is only a basic surface-level check - # todo: recursive dependency graph with networkx or topological sort? - all_modules = list(set(scanner._scan_modules + scanner._internal_modules + scanner._output_modules)) - while 1: - changed = False - dep_choices = CORE.module_loader.recommend_dependencies(all_modules) - if not dep_choices: - break - for event_type, deps in dep_choices.items(): - if event_type in ("*", "all"): - continue - # skip resolving dependency if a target provides the missing type - if any(e.type == event_type for e in scanner.target.events): - continue - required_by = deps.get("required_by", []) - recommended = deps.get("recommended", []) - if not recommended: - log.hugewarning( - f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no modules produce it" - ) - elif len(recommended) == 1: - log.verbose( - f"Enabling {next(iter(recommended))} because {len(required_by):,} modules ({','.join(required_by)}) rely on it for {event_type}" - ) - all_modules = list(set(all_modules + list(recommended))) - scanner._scan_modules = list(set(scanner._scan_modules + list(recommended))) - changed = True - else: - log.hugewarning( - f"{len(required_by):,} modules ({','.join(required_by)}) rely on {event_type} but no enabled module produces it" - ) - log.hugewarning( - f"Recommend enabling one or more of the following modules which produce {event_type}:" - ) - for m in recommended: - log.warning(f" - {m}") - if not changed: - break - - # required flags - modules = set(scanner._scan_modules) - for m in scanner._scan_modules: - flags = CORE.module_loader._preloaded.get(m, {}).get("flags", []) - if not all(f in flags for f in options.require_flags): - log.verbose( - f"Removing {m} because it does not have the required flags: {'+'.join(options.require_flags)}" - ) - with suppress(KeyError): - modules.remove(m) - - # excluded flags - for m in scanner._scan_modules: - flags = CORE.module_loader._preloaded.get(m, {}).get("flags", []) - if any(f in flags for f in options.exclude_flags): - log.verbose(f"Removing {m} because of excluded flag: {','.join(options.exclude_flags)}") - with suppress(KeyError): - modules.remove(m) - - # excluded modules - for m in options.exclude_modules: - if m in modules: - log.verbose(f"Removing {m} because it is excluded") - with suppress(KeyError): - modules.remove(m) - scanner._scan_modules = list(modules) - - log_fn = log.info - if options.list_modules or options.help_all: - log_fn = log.stdout - - help_modules = list(modules) - if module_filtering: - help_modules = None - - if options.help_all: - log_fn(CORE.args.parser.format_help()) - - if options.list_flags: - log.stdout("") - log.stdout("### FLAGS ###") - log.stdout("") - for row in CORE.module_loader.flags_table(flags=options.flags).splitlines(): - log.stdout(row) - return - - log_fn("") - log_fn("### MODULES ###") - log_fn("") - for row in CORE.module_loader.modules_table(modules=help_modules).splitlines(): - log_fn(row) - - if options.help_all: - log_fn("") - log_fn("### MODULE OPTIONS ###") - log_fn("") - for row in CORE.module_loader.modules_options_table(modules=help_modules).splitlines(): - log_fn(row) - - if options.list_modules or options.list_flags or options.help_all: - return - - module_list = CORE.module_loader.filter_modules(modules=modules) - deadly_modules = [] - active_modules = [] - active_aggressive_modules = [] - slow_modules = [] - for m in module_list: - if m[0] in scanner._scan_modules: - if "deadly" in m[-1]["flags"]: - deadly_modules.append(m[0]) - if "active" in m[-1]["flags"]: - active_modules.append(m[0]) - if "aggressive" in m[-1]["flags"]: - active_aggressive_modules.append(m[0]) - if "slow" in m[-1]["flags"]: - slow_modules.append(m[0]) - if scanner._scan_modules: - if deadly_modules and not options.allow_deadly: - log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") - log.hugewarning(f"Deadly modules are highly intrusive") - log.hugewarning(f"Please specify --allow-deadly to continue") - return False - if active_modules: - if active_modules: - if active_aggressive_modules: - log.hugewarning( - "This is an (aggressive) active scan! Intrusive connections will be made to target" - ) - else: - log.hugewarning( - "This is a (safe) active scan. Non-intrusive connections will be made to target" - ) - else: - log.hugeinfo("This is a passive scan. No connections will be made to target") - if slow_modules: - log.warning( - f"You have enabled the following slow modules: {','.join(slow_modules)}. Scan may take a while" - ) - - scanner.helpers.word_cloud.load() - - await scanner._prep() - - if not options.dry_run: - log.trace(f"Command: {' '.join(sys.argv)}") - - # if we're on the terminal, enable keyboard interaction - if sys.stdin.isatty(): - - import fcntl - from bbot.core.helpers.misc import smart_decode - - if not options.agent_mode and not options.yes: - log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}") - input() - - def handle_keyboard_input(keyboard_input): - kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") - if keyboard_input: - log.verbose(f'Got keyboard input: "{keyboard_input}"') - kill_match = kill_regex.match(keyboard_input) - if kill_match: - module = kill_match.group("module") - if module in scanner.modules: - log.hugewarning(f'Killing module: "{module}"') - scanner.manager.kill_module(module, message="killed by user") - else: - log.warning(f'Invalid module: "{module}"') - else: - CORE.logger.toggle_log_level(logger=log) - scanner.manager.modules_status(_log=True) - - reader = asyncio.StreamReader() - protocol = asyncio.StreamReaderProtocol(reader) - await asyncio.get_event_loop().connect_read_pipe(lambda: protocol, sys.stdin) - - # set stdout and stderr to blocking mode - # this is needed to prevent BlockingIOErrors in logging etc. - fds = [sys.stdout.fileno(), sys.stderr.fileno()] - for fd in fds: - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) - - async def akeyboard_listen(): - try: - allowed_errors = 10 - while 1: - keyboard_input = None - try: - keyboard_input = smart_decode((await reader.readline()).strip()) - allowed_errors = 10 - except Exception as e: - log_to_stderr(f"Error in keyboard listen loop: {e}", level="TRACE") - log_to_stderr(traceback.format_exc(), level="TRACE") - allowed_errors -= 1 - if keyboard_input is not None: - handle_keyboard_input(keyboard_input) - if allowed_errors <= 0: - break - except Exception as e: - log_to_stderr(f"Error in keyboard listen task: {e}", level="ERROR") - log_to_stderr(traceback.format_exc(), level="TRACE") - - asyncio.create_task(akeyboard_listen()) - - await scanner.async_start_without_generator() - - except errors.ScanError as e: - log_to_stderr(str(e), level="ERROR") - except Exception: - raise - - except errors.BBOTError as e: - log_to_stderr(f"{e} (--debug for details)", level="ERROR") - if CORE.logger.log_level <= logging.DEBUG: - log_to_stderr(traceback.format_exc(), level="DEBUG") - err = True - - except Exception: - log_to_stderr(f"Encountered unknown error: {traceback.format_exc()}", level="ERROR") - err = True - - finally: - # save word cloud - with suppress(BaseException): - save_success, filename = scanner.helpers.word_cloud.save() - if save_success: - log_to_stderr(f"Saved word cloud ({len(scanner.helpers.word_cloud):,} words) to {filename}") - # remove output directory if empty - with suppress(BaseException): - scanner.home.rmdir() - if err: - os._exit(1) + await scan.async_start_without_generator() def main(): diff --git a/bbot/core/core.py b/bbot/core/core.py index f88f2bfaf..d1dbcf42a 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,5 +1,7 @@ from pathlib import Path +from contextlib import suppress from omegaconf import OmegaConf +from omegaconf.errors import ConfigKeyError class BBOTCore: @@ -9,7 +11,6 @@ def __init__(self): self._files_config = None self.bbot_sudo_pass = None - self.cli_execution = False self._config = None self._default_config = None @@ -98,6 +99,10 @@ def custom_config(self, value): self._config = None self._custom_config = value + def del_config_item(self, item): + with suppress(ConfigKeyError): + del self.custom_config[item] + def merge_custom(self, config): self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 4e02c0e75..30163ab70 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -88,14 +88,23 @@ def parsed(self): def preset_from_args(self): args_preset = self.preset.__class__() + # scope args_preset.target.add_target(self.parsed.targets) args_preset.whitelist.add_target(self.parsed.whitelist) args_preset.blacklist.add_target(self.parsed.blacklist) args_preset.strict_scope = self.parsed.strict_scope + # modules args_preset.scan_modules = self.parsed.modules args_preset.output_modules = self.parsed.output_modules + args_preset.exclude_modules = self.parsed.exclude_modules + + # flags + args_preset.flags = self.parsed.flags + args_preset.require_flags = self.parsed.require_flags + args_preset.exclude_flags = self.parsed.exclude_flags + # additional custom presets / config options for preset_param in self.parsed.preset: if Path(preset_param).is_file(): @@ -112,6 +121,11 @@ def preset_from_args(self): log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") sys.exit(2) args_preset.core.merge_custom(cli_config) + + # verbosity levels + args_preset.silent = self.parsed.silent + args_preset.verbose = self.parsed.verbose + args_preset.debug = self.parsed.debug return args_preset def create_parser(self, *args, **kwargs): diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 015e7c07a..61cfb1777 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,4 +1,5 @@ import yaml +from copy import copy from pathlib import Path from omegaconf import OmegaConf @@ -21,6 +22,13 @@ def __init__( blacklist=None, modules=None, output_modules=None, + exclude_modules=None, + flags=None, + require_flags=None, + exclude_flags=None, + verbose=False, + debug=False, + silent=False, config=None, strict_scope=False, ): @@ -30,7 +38,7 @@ def __init__( self._module_loader = None # bbot core config - self.core = CORE + self.core = copy(CORE) if config is None: config = OmegaConf.create({}) # merge any custom configs @@ -45,8 +53,14 @@ def __init__( modules = [modules] if isinstance(output_modules, str): output_modules = [output_modules] - self.scan_modules = modules - self.output_modules = output_modules + self.scan_modules = set(modules if modules is not None else []) + self.output_modules = set(output_modules if output_modules is not None else []) + self.exclude_modules = set(exclude_modules if exclude_modules is not None else []) + + # module flags + self.flags = set(flags if flags is not None else []) + self.require_flags = set(require_flags if require_flags is not None else []) + self.exclude_flags = set(exclude_flags if exclude_flags is not None else []) # PRESET TODO: preparation of environment # self.core.environ.prepare() @@ -56,7 +70,7 @@ def __init__( # dirs to load modules from self.module_dirs = self.core.config.get("module_dirs", []) self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] - self.module_dirs = sorted(set(self.module_dirs)) + self.module_dirs = set(self.module_dirs) self.strict_scope = strict_scope @@ -72,6 +86,11 @@ def __init__( blacklist = [] self.blacklist = Target(self, *blacklist) + # log verbosity + self._verbose = verbose + self._debug = debug + self._silent = silent + self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() def merge(self, other): @@ -83,8 +102,13 @@ def merge(self, other): self.module_dirs = combined_module_dirs # TODO: refresh module dirs # modules - self.scan_modules = sorted(set(self.scan_modules).union(set(other.scan_modules))) - self.output_modules = sorted(set(self.output_modules).union(set(other.output_modules))) + self.scan_modules = set(self.scan_modules).union(set(other.scan_modules)) + self.output_modules = set(self.output_modules).union(set(other.output_modules)) + self.exclude_modules = set(self.exclude_modules).union(set(other.exclude_modules)) + # flags + self.flags = set(self.flags).union(set(other.flags)) + self.require_flags = set(self.require_flags).union(set(other.require_flags)) + self.exclude_flags = set(self.exclude_flags).union(set(other.exclude_flags)) # merge target / whitelist / blacklist self.target.add_target(other.target) self.whitelist.add_target(other.whitelist) @@ -93,6 +117,10 @@ def merge(self, other): self.strict_scope = self.strict_scope or other.strict_scope # config self.core.merge_custom(other.core.custom_config) + # log verbosity + self.silent = other.silent + self.verbose = other.verbose + self.debug = other.debug def parse_args(self): @@ -116,6 +144,51 @@ def parse_args(self): def config(self): return self.core.config + @property + def verbose(self): + return self._verbose + + @property + def debug(self): + return self._debug + + @property + def silent(self): + return self._silent + + @verbose.setter + def verbose(self, value): + if value: + self.debug = False + self.silent = False + self.core.merge_custom({"verbose": True}) + self.core.logger.set_log_level("VERBOSE") + else: + self.core.del_config_item("verbose") + self.core.logger.set_log_level("INFO") + + @debug.setter + def debug(self, value): + if value: + self.verbose = False + self.silent = False + self.core.merge_custom({"debug": True}) + self.core.logger.set_log_level("DEBUG") + else: + self.core.del_config_item("debug") + self.core.logger.set_log_level("INFO") + + @silent.setter + def silent(self, value): + if value: + self.verbose = False + self.debug = False + self.core.merge_custom({"silent": True}) + self.core.logger.set_log_level("CRITICAL") + else: + self.core.del_config_item("silent") + self.core.logger.set_log_level("INFO") + @property def helpers(self): if self._helpers is None: @@ -153,7 +226,7 @@ def internal_modules(self): @property def all_modules(self): - return sorted(set(self.scan_modules + self.output_modules + self.internal_modules)) + return sorted(self.scan_modules.union(self.output_modules).union(self.internal_modules)) @property def environ(self): @@ -208,34 +281,70 @@ def from_yaml(cls, yaml_preset): else: preset_dict = OmegaConf.create(yaml_preset) new_preset = cls( - *preset_dict.get("targets", []), - whitelist=preset_dict.get("whitelist", []), - blacklist=preset_dict.get("blacklist", []), - modules=preset_dict.get("scan_modules", []), - output_modules=preset_dict.get("output_modules", []), - config=preset_dict.get("config", None), + *preset_dict.get("targets"), + whitelist=preset_dict.get("whitelist"), + blacklist=preset_dict.get("blacklist"), + modules=preset_dict.get("scan_modules"), + output_modules=preset_dict.get("output_modules"), + exclude_modules=preset_dict.get("exclude_modules"), + flags=preset_dict.get("flags"), + require_flags=preset_dict.get("require_flags"), + exclude_flags=preset_dict.get("exclude_flags"), + verbose=preset_dict.get("verbose", False), + debug=preset_dict.get("debug", False), + silent=preset_dict.get("silent", False), + config=preset_dict.get("config"), strict_scope=preset_dict.get("strict_scope", False), ) return new_preset def to_yaml(self, full_config=False, sort_keys=False): + preset_dict = {} + + # config if full_config: config = self.core.config else: config = self.core.custom_config + config = OmegaConf.to_container(config) + if config: + preset_dict["config"] = config + + # scope target = sorted(str(t.data) for t in self.target) whitelist = sorted(str(t.data) for t in self.whitelist) blacklist = sorted(str(t.data) for t in self.blacklist) - preset_dict = { - "target": target, - "config": OmegaConf.to_container(config), - "modules": self.scan_modules, - "output_modules": self.output_modules, - } + if target: + preset_dict["target"] = target if whitelist and whitelist != target: preset_dict["whitelist"] = whitelist if blacklist: preset_dict["blacklist"] = blacklist if self.strict_scope: preset_dict["strict_scope"] = True + + # modules + if self.scan_modules: + preset_dict["modules"] = sorted(self.scan_modules) + if self.output_modules: + preset_dict["output_modules"] = sorted(self.output_modules) + if self.exclude_modules: + preset_dict["exclude_modules"] = sorted(self.exclude_modules) + + # flags + if self.flags: + preset_dict["flags"] = sorted(self.flags) + if self.require_flags: + preset_dict["require_flags"] = sorted(self.require_flags) + if self.exclude_flags: + preset_dict["exclude_flags"] = sorted(self.exclude_flags) + + # log verbosity + if self.verbose: + preset_dict["verbose"] = True + if self.debug: + preset_dict["debug"] = True + if self.silent: + preset_dict["silent"] = True + return yaml.dump(preset_dict, sort_keys=sort_keys) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index f07bafebd..d22743601 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -128,11 +128,10 @@ def add_target(self, t, event_type=None): else: try: event = make_event( - single_target + single_target, event_type=event_type, - source=self.scan.root_event, - module=self._dummy_module, - tags=["target"], + dummy=True, + tags=["target"], ) except ValidationError as e: # allow commented lines diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 53971eecd..116bf98c8 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -1,21 +1,15 @@ def test_presets(): - from bbot.core import CORE + pass - assert "module_paths" in CORE.config - assert CORE.module_loader.module_paths - assert any(str(x).endswith("/modules") for x in CORE.module_loader.module_paths) - assert "HTTP_RESPONSE" in CORE.config.omit_event_types + # test config merging - # make sure .copy() works as intended - # preset_copy = CORE.copy() - # assert isinstance(preset_copy, CORE.__class__) - # base_tests(CORE) - # base_tests(preset_copy) - # preset_copy.update({"asdf": {"fdsa": "1234"}}) - # assert "asdf" in preset_copy - # assert preset_copy.asdf.fdsa == "1234" - # assert not "asdf" in CORE + # make sure custom / default split works as expected - # preset_copy["testing"] = {"test1": "value"} - # assert "testing" in preset_copy - # assert "testing" not in CORE + # test preset merging + + # test verbosity levels (conflicting verbose/debug/silent) + + # test custom module load directory + # make sure it works with cli arg module/flag/config syntax validation + + # test yaml save/load, make sure it's the same From 52b05e09bd6c1f3568f99b16f7c465be567141ad Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 12 Mar 2024 10:39:40 -0400 Subject: [PATCH 024/171] add basic tests --- bbot/scanner/__init__.py | 1 + bbot/scanner/preset/preset.py | 29 +++++--- bbot/test/bbot_fixtures.py | 21 +++--- bbot/test/test.conf | 101 +++++++++++++------------- bbot/test/test_step_1/test_presets.py | 52 ++++++++++++- 5 files changed, 129 insertions(+), 75 deletions(-) diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py index cc993af8a..1622f4c20 100644 --- a/bbot/scanner/__init__.py +++ b/bbot/scanner/__init__.py @@ -1 +1,2 @@ +from .preset import Preset from .scanner import Scanner diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 61cfb1777..e8bde6643 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -281,10 +281,10 @@ def from_yaml(cls, yaml_preset): else: preset_dict = OmegaConf.create(yaml_preset) new_preset = cls( - *preset_dict.get("targets"), + *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), blacklist=preset_dict.get("blacklist"), - modules=preset_dict.get("scan_modules"), + modules=preset_dict.get("modules"), output_modules=preset_dict.get("output_modules"), exclude_modules=preset_dict.get("exclude_modules"), flags=preset_dict.get("flags"), @@ -298,7 +298,7 @@ def from_yaml(cls, yaml_preset): ) return new_preset - def to_yaml(self, full_config=False, sort_keys=False): + def to_dict(self, include_target=False, full_config=False): preset_dict = {} # config @@ -311,15 +311,16 @@ def to_yaml(self, full_config=False, sort_keys=False): preset_dict["config"] = config # scope - target = sorted(str(t.data) for t in self.target) - whitelist = sorted(str(t.data) for t in self.whitelist) - blacklist = sorted(str(t.data) for t in self.blacklist) - if target: - preset_dict["target"] = target - if whitelist and whitelist != target: - preset_dict["whitelist"] = whitelist - if blacklist: - preset_dict["blacklist"] = blacklist + if include_target: + target = sorted(str(t.data) for t in self.target) + whitelist = sorted(str(t.data) for t in self.whitelist) + blacklist = sorted(str(t.data) for t in self.blacklist) + if target: + preset_dict["target"] = target + if whitelist and whitelist != target: + preset_dict["whitelist"] = whitelist + if blacklist: + preset_dict["blacklist"] = blacklist if self.strict_scope: preset_dict["strict_scope"] = True @@ -347,4 +348,8 @@ def to_yaml(self, full_config=False, sort_keys=False): if self.silent: preset_dict["silent"] = True + return preset_dict + + def to_yaml(self, include_target=False, full_config=False, sort_keys=False): + preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 72766843a..36d9ee889 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -225,12 +225,15 @@ def agent(monkeypatch, bbot_config): # bbot config -from bbot.core import CORE +from bbot.scanner import Preset + +default_preset = Preset.from_yaml(Path(__file__).parent / "test.conf") +test_config = default_preset.config + +available_modules = list(default_preset.module_loader.configs(type="scan")) +available_output_modules = list(default_preset.module_loader.configs(type="output")) +available_internal_modules = list(default_preset.module_loader.configs(type="internal")) -# PRESET TODO: revisit this -default_config = CORE.config -test_config = OmegaConf.load(Path(__file__).parent / "test.conf") -test_config = OmegaConf.merge(default_config, test_config) if test_config.get("debug", False): logging.getLogger("bbot").setLevel(logging.DEBUG) @@ -241,16 +244,10 @@ def bbot_config(): return test_config -# PRESET TODO: revisit this -available_modules = list(CORE.module_loader.configs(type="scan")) -available_output_modules = list(CORE.module_loader.configs(type="output")) -available_internal_modules = list(CORE.module_loader.configs(type="internal")) - - @pytest.fixture(autouse=True) def install_all_python_deps(): deps_pip = set() - for module in module_loader.preloaded().values(): + for module in default_preset.module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test.conf b/bbot/test/test.conf index ba8367461..1c6e18750 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -1,51 +1,52 @@ home: /tmp/.bbot_test -modules: - massdns: - wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt - ffuf: - prefix_busting: true - ipneighbor: - test_option: ipneighbor -output_modules: - http: - url: http://127.0.0.1:11111 - username: username - password: password - bearer: bearer - websocket: - url: ws://127.0.0.1/ws:11111 - token: asdf - neo4j: - uri: bolt://127.0.0.1:11111 - python: - test_option: asdf -internal_modules: - speculate: - test_option: speculate -http_proxy: -http_headers: { "test": "header" } -ssl_verify: false -scope_search_distance: 0 -scope_report_distance: 0 -scope_dns_search_distance: 1 -plumbus: asdf -dns_debug: false -user_agent: "BBOT Test User-Agent" -http_debug: false -agent_url: ws://127.0.0.1:8765 -agent_token: test -dns_resolution: false -dns_timeout: 1 -speculate: false -excavate: false -aggregate: false -omit_event_types: [] -debug: true -dns_wildcard_ignore: - - blacklanternsecurity.com - - fakedomain - - notreal - - google - - google.com - - example.com - - evilcorp.com +config: + modules: + massdns: + wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt + ffuf: + prefix_busting: true + ipneighbor: + test_option: ipneighbor + output_modules: + http: + url: http://127.0.0.1:11111 + username: username + password: password + bearer: bearer + websocket: + url: ws://127.0.0.1/ws:11111 + token: asdf + neo4j: + uri: bolt://127.0.0.1:11111 + python: + test_option: asdf + internal_modules: + speculate: + test_option: speculate + http_proxy: + http_headers: { "test": "header" } + ssl_verify: false + scope_search_distance: 0 + scope_report_distance: 0 + scope_dns_search_distance: 1 + plumbus: asdf + dns_debug: false + user_agent: "BBOT Test User-Agent" + http_debug: false + agent_url: ws://127.0.0.1:8765 + agent_token: test + dns_resolution: false + dns_timeout: 1 + speculate: false + excavate: false + aggregate: false + omit_event_types: [] + debug: true + dns_wildcard_ignore: + - blacklanternsecurity.com + - fakedomain + - notreal + - google + - google.com + - example.com + - evilcorp.com diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 116bf98c8..2f7177342 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -1,5 +1,55 @@ +from ..bbot_fixtures import * # noqa F401 + +from bbot.scanner import Preset + + def test_presets(): - pass + + blank_preset = Preset() + assert not blank_preset.target + assert blank_preset.strict_scope == False + + preset1 = Preset( + "evilcorp.com", + "www.evilcorp.ce", + whitelist=["evilcorp.ce"], + blacklist=["test.www.evilcorp.ce"], + modules=["sslcert"], + output_modules=["json"], + exclude_modules=["ipneighbor"], + flags=["subdomain-enum"], + require_flags=["safe"], + exclude_flags=["slow"], + verbose=False, + debug=False, + silent=True, + config={"preset_test_asdf": 1}, + strict_scope=True, + ) + + # test yaml save/load + yaml1 = preset1.to_yaml(sort_keys=True) + preset2 = Preset.from_yaml(yaml1) + yaml2 = preset2.to_yaml(sort_keys=True) + assert yaml1 == yaml2 + + # test preset merging + # preset3 = Preset( + # "evilcorp.org", + # whitelist=["evilcorp.ce"], + # blacklist=["test.www.evilcorp.ce"], + # modules=["sslcert"], + # output_modules=["json"], + # exclude_modules=["ipneighbor"], + # flags=["subdomain-enum"], + # require_flags=["safe"], + # exclude_flags=["slow"], + # verbose=False, + # debug=False, + # silent=True, + # config={"preset_test_asdf": 1}, + # strict_scope=True, + # ) # test config merging From 670d9f8b592bce0551f6cc44b2fe03a827c285da Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 12 Mar 2024 12:41:33 -0400 Subject: [PATCH 025/171] fix parser error --- bbot/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 838a64280..69d056262 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -25,6 +25,8 @@ async def _main(): # start by creating a default scan preset preset = Preset() + # parse command line arguments and merge into preset + preset.parse_args() # print help if no arguments if len(sys.argv) == 1: @@ -32,9 +34,6 @@ async def _main(): sys.exit(1) return - # parse command line arguments and merge into preset - preset.parse_args() - # --version if preset.args.parsed.version: log.stdout(__version__) From 8f95ce632e904873af3d3b9707ba250b62af33fe Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Tue, 12 Mar 2024 13:18:01 -0500 Subject: [PATCH 026/171] Fixed log_level Error --- bbot/scanner/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index fcb961625..ff10fc85f 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -904,7 +904,7 @@ def log_level(self): """ Return the current log level, e.g. logging.INFO """ - return self.core.log_level + return self.core.logger.log_level @property def _log_handlers(self): From ab933c5c07f6917e9c8d487dedf6f66bc227fd0a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 12 Mar 2024 17:36:57 -0400 Subject: [PATCH 027/171] more tests --- bbot/scanner/manager.py | 1 + bbot/scanner/preset/preset.py | 13 ++-- bbot/scanner/target.py | 28 +++----- bbot/test/test_step_1/test_presets.py | 92 ++++++++++++++++++++------- 4 files changed, 85 insertions(+), 49 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 0fd95a0a0..ec124b9c4 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -73,6 +73,7 @@ async def init_events(self): sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) for event in sorted_events: event._dummy = False + event.scope_distance = 0 event.scan = self.scan event.source = self.scan.root_event event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index e8bde6643..9cbf57dac 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -77,14 +77,14 @@ def __init__( # target / whitelist / blacklist from bbot.scanner.target import Target - self.target = Target(self, *targets, strict_scope=strict_scope, make_in_scope=True) + self.target = Target(*targets, strict_scope=self.strict_scope) if not whitelist: self.whitelist = self.target.copy() else: - self.whitelist = Target(self, *whitelist, strict_scope=self.strict_scope) + self.whitelist = Target(*whitelist, strict_scope=self.strict_scope) if not blacklist: blacklist = [] - self.blacklist = Target(self, *blacklist) + self.blacklist = Target(*blacklist) # log verbosity self._verbose = verbose @@ -109,12 +109,13 @@ def merge(self, other): self.flags = set(self.flags).union(set(other.flags)) self.require_flags = set(self.require_flags).union(set(other.require_flags)) self.exclude_flags = set(self.exclude_flags).union(set(other.exclude_flags)) - # merge target / whitelist / blacklist + # scope self.target.add_target(other.target) - self.whitelist.add_target(other.whitelist) + self.whitelist = other.whitelist self.blacklist.add_target(other.blacklist) - # scope self.strict_scope = self.strict_scope or other.strict_scope + for t in (self.target, self.whitelist): + t.strict_scope = self.strict_scope # config self.core.merge_custom(other.core.custom_config) # log verbosity diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index d22743601..77958ff17 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -5,6 +5,7 @@ from bbot.core.errors import * from bbot.modules.base import BaseModule from bbot.core.event import make_event, is_event +from bbot.core.helpers.misc import ip_network_parents, is_ip_type, domain_parents log = logging.getLogger("bbot.core.target") @@ -14,12 +15,11 @@ class Target: A class representing a target. Can contain an unlimited number of hosts, IP or IP ranges, URLs, etc. Attributes: - make_in_scope (bool): Specifies whether to mark contained events as in-scope. - scan (Scan): Reference to the Scan object that instantiated the Target. - _events (dict): Dictionary mapping hosts to events related to the target. strict_scope (bool): Flag indicating whether to consider child domains in-scope. If set to True, only the exact hosts specified and not their children are considered part of the target. + _events (dict): Dictionary mapping hosts to events related to the target. + Examples: Basic usage >>> target = Target(scan, "evilcorp.com", "1.2.3.0/24") @@ -62,17 +62,13 @@ class Target: - If you do not want to include child subdomains, use `strict_scope=True` """ - def __init__(self, preset, *targets, strict_scope=False, make_in_scope=False): + def __init__(self, *targets, strict_scope=False): """ Initialize a Target object. Args: scan (Scan): Reference to the Scan object that instantiated the Target. *targets: One or more targets (e.g., domain names, IP ranges) to be included in this Target. - strict_scope (bool, optional): Flag to control whether only the exact hosts are considered in-scope. - Defaults to False. - make_in_scope (bool, optional): Flag to control whether contained events are marked as in-scope. - Defaults to False. Attributes: scan (Scan): Reference to the Scan object. @@ -83,10 +79,7 @@ def __init__(self, preset, *targets, strict_scope=False, make_in_scope=False): - The strict_scope flag can be set to restrict scope calculation to only exactly-matching hosts and not their child subdomains. - Each target is processed and stored as an `Event` in the '_events' dictionary. """ - self.preset = preset self.strict_scope = strict_scope - self.make_in_scope = make_in_scope - self._events = dict() if len(targets) > 0: log.verbose(f"Creating events from {len(targets):,} targets") @@ -111,7 +104,6 @@ def add_target(self, t, event_type=None): Notes: - If `t` is of the same class as this Target, all its events are merged. - If `t` is an event, it is directly added to `_events`. - - If `make_in_scope` is True, the scope distance of the event is set to 0. """ if not isinstance(t, (list, tuple, set)): t = [t] @@ -137,8 +129,6 @@ def add_target(self, t, event_type=None): # allow commented lines if not str(t).startswith("#"): raise ValidationError(f'Could not add target "{t}": {e}') - if self.make_in_scope: - event.scope_distance = 0 try: self._events[event.host].add(event) except KeyError: @@ -171,7 +161,7 @@ def copy(self): Creates and returns a copy of the Target object, including a shallow copy of the `_events` attribute. Returns: - Target: A new Target object with the same `scan` and `strict_scope` attributes as the original. + Target: A new Target object with the sameattributes as the original. A shallow copy of the `_events` dictionary is made. Examples: @@ -189,7 +179,7 @@ def copy(self): Notes: - The `scan` object reference is kept intact in the copied Target object. """ - self_copy = self.__class__(self.preset, strict_scope=self.strict_scope) + self_copy = self.__class__() self_copy._events = dict(self._events) return self_copy @@ -222,12 +212,12 @@ def get(self, host): if other.host: with suppress(KeyError, StopIteration): return next(iter(self._events[other.host])) - if self.preset.helpers.is_ip_type(other.host): - for n in self.preset.helpers.ip_network_parents(other.host, include_self=True): + if is_ip_type(other.host): + for n in ip_network_parents(other.host, include_self=True): with suppress(KeyError, StopIteration): return next(iter(self._events[n])) elif not self.strict_scope: - for h in self.preset.helpers.domain_parents(other.host): + for h in domain_parents(other.host): with suppress(KeyError, StopIteration): return next(iter(self._events[h])) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 2f7177342..cb272687f 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -3,11 +3,7 @@ from bbot.scanner import Preset -def test_presets(): - - blank_preset = Preset() - assert not blank_preset.target - assert blank_preset.strict_scope == False +def test_preset_yaml(): preset1 = Preset( "evilcorp.com", @@ -24,9 +20,45 @@ def test_presets(): debug=False, silent=True, config={"preset_test_asdf": 1}, - strict_scope=True, + strict_scope=False, + ) + assert "evilcorp.com" in preset1.target + assert "evilcorp.ce" in preset1.whitelist + assert "test.www.evilcorp.ce" in preset1.blacklist + assert "sslcert" in preset1.scan_modules + + # test yaml save/load + yaml1 = preset1.to_yaml(sort_keys=True) + preset2 = Preset.from_yaml(yaml1) + yaml2 = preset2.to_yaml(sort_keys=True) + assert yaml1 == yaml2 + + +def test_preset_scope(): + + blank_preset = Preset() + assert not blank_preset.target + assert blank_preset.strict_scope == False + + preset1 = Preset( + "evilcorp.com", + "www.evilcorp.ce", + whitelist=["evilcorp.ce"], + blacklist=["test.www.evilcorp.ce"], ) + # make sure target logic works as expected + assert "evilcorp.com" in preset1.target + assert "asdf.evilcorp.com" in preset1.target + assert "asdf.www.evilcorp.ce" in preset1.target + assert not "evilcorp.ce" in preset1.target + assert "evilcorp.ce" in preset1.whitelist + assert "test.www.evilcorp.ce" in preset1.blacklist + assert not "evilcorp.ce" in preset1.blacklist + assert preset1.in_scope("www.evilcorp.ce") + assert not preset1.in_scope("evilcorp.com") + assert not preset1.in_scope("asdf.test.www.evilcorp.ce") + # test yaml save/load yaml1 = preset1.to_yaml(sort_keys=True) preset2 = Preset.from_yaml(yaml1) @@ -34,22 +66,36 @@ def test_presets(): assert yaml1 == yaml2 # test preset merging - # preset3 = Preset( - # "evilcorp.org", - # whitelist=["evilcorp.ce"], - # blacklist=["test.www.evilcorp.ce"], - # modules=["sslcert"], - # output_modules=["json"], - # exclude_modules=["ipneighbor"], - # flags=["subdomain-enum"], - # require_flags=["safe"], - # exclude_flags=["slow"], - # verbose=False, - # debug=False, - # silent=True, - # config={"preset_test_asdf": 1}, - # strict_scope=True, - # ) + preset3 = Preset( + "evilcorp.org", + whitelist=["evilcorp.de"], + blacklist=["test.www.evilcorp.de"], + strict_scope=True, + ) + + preset1.merge(preset3) + + # targets should be merged + assert "evilcorp.com" in preset1.target + assert "www.evilcorp.ce" in preset1.target + assert "evilcorp.org" in preset1.target + # strict scope is enabled + assert not "asdf.evilcorp.com" in preset1.target + assert not "asdf.www.evilcorp.ce" in preset1.target + # whitelist is overridden, not merged + assert not "evilcorp.ce" in preset1.whitelist + assert "evilcorp.de" in preset1.whitelist + assert not "asdf.evilcorp.de" in preset1.whitelist + # blacklist should be merged, strict scope does not apply + assert "asdf.test.www.evilcorp.ce" in preset1.blacklist + assert "asdf.test.www.evilcorp.de" in preset1.blacklist + # only the base domain of evilcorp.de should be in scope + assert not preset1.in_scope("evilcorp.com") + assert not preset1.in_scope("evilcorp.org") + assert preset1.in_scope("evilcorp.de") + assert not preset1.in_scope("asdf.evilcorp.de") + assert not preset1.in_scope("evilcorp.com") + assert not preset1.in_scope("asdf.test.www.evilcorp.ce") # test config merging @@ -61,5 +107,3 @@ def test_presets(): # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation - - # test yaml save/load, make sure it's the same From b489b7be9f8260bd2a8e2ea5e3219e13e7525c5c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 13 Mar 2024 09:44:51 -0400 Subject: [PATCH 028/171] better logging + tests --- bbot/core/config/logger.py | 35 ++++++++-------- bbot/core/core.py | 49 +++++++--------------- bbot/scanner/preset/preset.py | 55 ++++++++++++++---------- bbot/test/test_step_1/test_presets.py | 60 ++++++++++++++++++++++++--- 4 files changed, 121 insertions(+), 78 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index d72a59395..72a462465 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -1,4 +1,3 @@ -import os import sys import logging from copy import copy @@ -51,7 +50,7 @@ def __init__(self, core): self._loggers = None self._log_handlers = None - self._log_level_override = None + self._log_level = None self.core_logger = logging.getLogger("bbot") self.core = core @@ -60,6 +59,13 @@ def __init__(self, core): for logger in self.loggers: self.include_logger(logger) + if core.config.get("verbose", False): + self.log_level = logging.VERBOSE + elif core.config.get("debug", False): + self.log_level = logging.DEBUG + else: + self.log_level = logging.INFO + def addLoggingLevel(self, levelName, levelNum, methodName=None): """ Comprehensively adds a new logging level to the `logging` module and the @@ -134,7 +140,8 @@ def remove_log_handler(self, handler): def include_logger(self, logger): if logger not in self.loggers: self.loggers.append(logger) - logger.setLevel(self.log_level) + if self.log_level is not None: + logger.setLevel(self.log_level) for handler in self.log_handlers.values(): logger.addHandler(handler) @@ -194,28 +201,20 @@ def stderr_filter(record): @property def log_level(self): - if self._log_level_override is not None: - return self._log_level_override - - if self.core.config.get("debug", False) or os.environ.get("BBOT_DEBUG", "").lower() in ("true", "yes"): - return logging.DEBUG + if self._log_level is None: + return logging.INFO + return self._log_level - loglevel = logging.INFO - # PRESET TODO: delete / move this - # if self.core.cli_execution: - # if self.core.args.parsed.verbose: - # loglevel = logging.VERBOSE - # if self.core.args.parsed.debug: - # loglevel = logging.DEBUG - return loglevel + @log_level.setter + def log_level(self, level): + self.set_log_level(level) def set_log_level(self, level, logger=None): if isinstance(level, str): level = logging.getLevelName(level) if logger is not None: logger.hugeinfo(f"Setting log level to {logging.getLevelName(level)}") - self.core.custom_config["silent"] = False - self._log_level_override = level + self._log_level = level for logger in self.loggers: logger.setLevel(level) diff --git a/bbot/core/core.py b/bbot/core/core.py index d1dbcf42a..f87567b32 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,7 +1,6 @@ +from copy import copy from pathlib import Path -from contextlib import suppress from omegaconf import OmegaConf -from omegaconf.errors import ConfigKeyError class BBOTCore: @@ -31,33 +30,6 @@ def __init__(self): # - check_cli_args # - ensure_config_files - # first, we load config files - # - ensure bbot home directory (needed for logging) - # - ensure module load directories (needed for preloading modules) - - ### to save on performance, we stop here - ### the rest of the attributes populate lazily only when accessed - ### we do this to minimize the time it takes to import bbot as a code library - - # next, we preload modules (needed for parsing CLI args) - # self.load_module_configs() - - # next, we load environment variables - # todo: automatically propagate config values to environ? (would require __setitem__ hooks) - # self.load_environ() - - # finally, we parse CLI args - # self.parse_cli_args() - - @property - def files_config(self): - if self._files_config is None: - from .config import files - - self.files = files - self._files_config = files.BBOTConfigFiles(self) - return self._files_config - @property def config(self): """ @@ -99,16 +71,27 @@ def custom_config(self, value): self._config = None self._custom_config = value - def del_config_item(self, item): - with suppress(ConfigKeyError): - del self.custom_config[item] - def merge_custom(self, config): self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) def merge_default(self, config): self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config)) + def copy(self): + core_copy = copy(self) + core_copy._default_config = self._default_config.copy() + core_copy._custom_config = self._custom_config.copy() + return core_copy + + @property + def files_config(self): + if self._files_config is None: + from .config import files + + self.files = files + self._files_config = files.BBOTConfigFiles(self) + return self._files_config + @property def logger(self): self.config diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9cbf57dac..4d037a928 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,7 +1,7 @@ import yaml -from copy import copy +import omegaconf from pathlib import Path -from omegaconf import OmegaConf +from contextlib import suppress from bbot.core import CORE from bbot.core.event.base import make_event @@ -37,10 +37,14 @@ def __init__( self._helpers = None self._module_loader = None + self._verbose = False + self._debug = False + self._silent = False + # bbot core config - self.core = copy(CORE) + self.core = CORE.copy() if config is None: - config = OmegaConf.create({}) + config = omegaconf.OmegaConf.create({}) # merge any custom configs self.core.merge_custom(config) @@ -64,8 +68,6 @@ def __init__( # PRESET TODO: preparation of environment # self.core.environ.prepare() - if self.core.config.get("debug", False): - self.core.logger.set_log_level("DEBUG") # dirs to load modules from self.module_dirs = self.core.config.get("module_dirs", []) @@ -87,9 +89,12 @@ def __init__( self.blacklist = Target(*blacklist) # log verbosity - self._verbose = verbose - self._debug = debug - self._silent = silent + if verbose: + self.verbose = verbose + if debug: + self.debug = debug + if silent: + self.silent = silent self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() @@ -163,10 +168,12 @@ def verbose(self, value): self.debug = False self.silent = False self.core.merge_custom({"verbose": True}) - self.core.logger.set_log_level("VERBOSE") + self.core.logger.log_level = "VERBOSE" else: - self.core.del_config_item("verbose") - self.core.logger.set_log_level("INFO") + with suppress(omegaconf.errors.ConfigKeyError): + del self.core.custom_config["verbose"] + self.core.logger.log_level = "INFO" + self._verbose = value @debug.setter def debug(self, value): @@ -174,10 +181,12 @@ def debug(self, value): self.verbose = False self.silent = False self.core.merge_custom({"debug": True}) - self.core.logger.set_log_level("DEBUG") + self.core.logger.log_level = "DEBUG" else: - self.core.del_config_item("debug") - self.core.logger.set_log_level("INFO") + with suppress(omegaconf.errors.ConfigKeyError): + del self.core.custom_config["debug"] + self.core.logger.log_level = "INFO" + self._debug = value @silent.setter def silent(self, value): @@ -185,10 +194,12 @@ def silent(self, value): self.verbose = False self.debug = False self.core.merge_custom({"silent": True}) - self.core.logger.set_log_level("CRITICAL") + self.core.logger.log_level = "CRITICAL" else: - self.core.del_config_item("silent") - self.core.logger.set_log_level("INFO") + with suppress(omegaconf.errors.ConfigKeyError): + del self.core.custom_config["silent"] + self.core.logger.log_level = "INFO" + self._silent = value @property def helpers(self): @@ -210,7 +221,7 @@ def module_loader(self): self._module_loader = ModuleLoader(self) # update default config with module defaults - module_config = OmegaConf.create( + module_config = omegaconf.OmegaConf.create( { "modules": self._module_loader.configs(type="scan"), "output_modules": self._module_loader.configs(type="output"), @@ -278,9 +289,9 @@ def whitelisted(self, e): @classmethod def from_yaml(cls, yaml_preset): if Path(yaml_preset).is_file(): - preset_dict = OmegaConf.load(yaml_preset) + preset_dict = omegaconf.OmegaConf.load(yaml_preset) else: - preset_dict = OmegaConf.create(yaml_preset) + preset_dict = omegaconf.OmegaConf.create(yaml_preset) new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -307,7 +318,7 @@ def to_dict(self, include_target=False, full_config=False): config = self.core.config else: config = self.core.custom_config - config = OmegaConf.to_container(config) + config = omegaconf.OmegaConf.to_container(config) if config: preset_dict["config"] = config diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index cb272687f..27ed0a95d 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -3,6 +3,51 @@ from bbot.scanner import Preset +def test_core(): + from bbot.core import CORE + + import omegaconf + + assert "testasdf" not in CORE.default_config + assert "testasdf" not in CORE.custom_config + assert "testasdf" not in CORE.config + + core_copy = CORE.copy() + # make sure our default config is read-only + with pytest.raises(omegaconf.errors.ReadonlyConfigError): + core_copy.default_config["testasdf"] = "test" + # same for merged config + with pytest.raises(omegaconf.errors.ReadonlyConfigError): + core_copy.config["testasdf"] = "test" + + assert "testasdf" not in core_copy.default_config + assert "testasdf" not in core_copy.custom_config + assert "testasdf" not in core_copy.config + + core_copy.custom_config["testasdf"] = "test" + assert "testasdf" not in core_copy.default_config + assert "testasdf" in core_copy.custom_config + assert "testasdf" in core_copy.config + + # test config merging + config_to_merge = omegaconf.OmegaConf.create({"test123": {"test321": [3, 2, 1], "test456": [4, 5, 6]}}) + core_copy.merge_custom(config_to_merge) + assert "test123" not in core_copy.default_config + assert "test123" in core_copy.custom_config + assert "test123" in core_copy.config + assert "test321" in core_copy.custom_config["test123"] + assert "test321" in core_copy.config["test123"] + + # test deletion + del core_copy.custom_config.test123.test321 + assert "test123" in core_copy.custom_config + assert "test123" in core_copy.config + assert "test321" not in core_copy.custom_config["test123"] + assert "test321" not in core_copy.config["test123"] + assert "test456" in core_copy.custom_config["test123"] + assert "test456" in core_copy.config["test123"] + + def test_preset_yaml(): preset1 = Preset( @@ -97,13 +142,18 @@ def test_preset_scope(): assert not preset1.in_scope("evilcorp.com") assert not preset1.in_scope("asdf.test.www.evilcorp.ce") - # test config merging - - # make sure custom / default split works as expected - - # test preset merging +def test_preset_misc(): # test verbosity levels (conflicting verbose/debug/silent) + preset = Preset(verbose=True) + assert preset.verbose == True + assert preset.debug == False + assert preset.silent == False + assert preset.core.logger.log_level == logging.VERBOSE + preset.debug = True + assert preset.verbose == False + assert preset.debug == True + assert preset.silent == False # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation From 20f1f602decfb580b4967ac95aa2cd367bae57c8 Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Wed, 13 Mar 2024 10:15:18 -0500 Subject: [PATCH 029/171] Fixed Default Log Location to go to Intended Folder in ~/.bbot/logs --- bbot/core/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 72a462465..03af61de8 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -148,7 +148,7 @@ def include_logger(self, logger): @property def log_handlers(self): if self._log_handlers is None: - log_dir = Path(self.core.config["home"]) / "logs" + log_dir = Path(self.core.home) / "logs" if not mkdir(log_dir, raise_error=False): error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})") From 35bf2dce49ae4822fa34988b2ed94fd6bd755e94 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 13 Mar 2024 10:21:15 -0400 Subject: [PATCH 030/171] implement module dependencies in preloading --- bbot/core/modules.py | 85 +++++++++++++++++++++++++++----------------- bbot/modules/base.py | 3 ++ 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 4e40bde97..81682f6e8 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -15,7 +15,7 @@ class ModuleLoader: """ - Main class responsible for loading BBOT modules. + Main class responsible for preloading BBOT modules. This class is in charge of preloading modules to determine their dependencies. Once dependencies are identified, they are installed before the actual module is imported. @@ -24,6 +24,9 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") + # if a module consumes these event types, automatically assume these dependencies + default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx"} + def __init__(self, preset): self.preset = preset self.__preloaded = None @@ -197,6 +200,9 @@ def preload_module(self, module_file): "options_desc": {}, "hash": "d5a88dd3866c876b81939c920bf4959716e2a374", "deps": { + "modules": [ + "httpx" + ] "pip": [ "python-Wappalyzer~=0.3.1" ], @@ -208,14 +214,15 @@ def preload_module(self, module_file): "sudo": false } """ - watched_events = [] - produced_events = [] - flags = [] + watched_events = set() + produced_events = set() + flags = set() meta = {} - pip_deps = [] - pip_deps_constraints = [] - shell_deps = [] - apt_deps = [] + deps_modules = set() + deps_pip = [] + deps_pip_constraints = [] + deps_shell = [] + deps_apt = [] ansible_tasks = [] python_code = open(module_file).read() # take a hash of the code so we can keep track of when it changes @@ -227,6 +234,7 @@ def preload_module(self, module_file): # look for classes if type(root_element) == ast.ClassDef: for class_attr in root_element.body: + # class attributes that are dictionaries if type(class_attr) == ast.Assign and type(class_attr.value) == ast.Dict: # module options @@ -238,68 +246,81 @@ def preload_module(self, module_file): # module metadata if any([target.id == "meta" for target in class_attr.targets]): meta = ast.literal_eval(class_attr.value) + # class attributes that are lists if type(class_attr) == ast.Assign and type(class_attr.value) == ast.List: # flags if any([target.id == "flags" for target in class_attr.targets]): for flag in class_attr.value.elts: if type(flag.value) == str: - flags.append(flag.value) + flags.add(flag.value) # watched events if any([target.id == "watched_events" for target in class_attr.targets]): for event_type in class_attr.value.elts: if type(event_type.value) == str: - watched_events.append(event_type.value) + watched_events.add(event_type.value) # produced events if any([target.id == "produced_events" for target in class_attr.targets]): for event_type in class_attr.value.elts: if type(event_type.value) == str: - produced_events.append(event_type.value) + produced_events.add(event_type.value) + + # bbot module dependencies + if any([target.id == "deps_modules" for target in class_attr.targets]): + for dep_module in class_attr.value.elts: + if type(dep_module.value) == str: + deps_modules.add(dep_module.value) # python dependencies if any([target.id == "deps_pip" for target in class_attr.targets]): - for python_dep in class_attr.value.elts: - if type(python_dep.value) == str: - pip_deps.append(python_dep.value) - + for dep_pip in class_attr.value.elts: + if type(dep_pip.value) == str: + deps_pip.append(dep_pip.value) if any([target.id == "deps_pip_constraints" for target in class_attr.targets]): - for python_dep in class_attr.value.elts: - if type(python_dep.value) == str: - pip_deps_constraints.append(python_dep.value) - + for dep_pip in class_attr.value.elts: + if type(dep_pip.value) == str: + deps_pip_constraints.append(dep_pip.value) # apt dependencies elif any([target.id == "deps_apt" for target in class_attr.targets]): - for apt_dep in class_attr.value.elts: - if type(apt_dep.value) == str: - apt_deps.append(apt_dep.value) + for dep_apt in class_attr.value.elts: + if type(dep_apt.value) == str: + deps_apt.append(dep_apt.value) # bash dependencies elif any([target.id == "deps_shell" for target in class_attr.targets]): - for shell_dep in class_attr.value.elts: - shell_deps.append(ast.literal_eval(shell_dep)) + for dep_shell in class_attr.value.elts: + deps_shell.append(ast.literal_eval(dep_shell)) # ansible playbook elif any([target.id == "deps_ansible" for target in class_attr.targets]): ansible_tasks = ast.literal_eval(class_attr.value) + for task in ansible_tasks: if not "become" in task: task["become"] = False # don't sudo brew elif os_platform() == "darwin" and ("package" in task and task.get("become", False) == True): task["become"] = False + + # derive module dependencies from watched event types + for event_type in watched_events: + if event_type in self.default_module_deps: + deps_modules.add(self.default_module_deps[event_type]) + preloaded_data = { - "watched_events": watched_events, - "produced_events": produced_events, - "flags": flags, + "watched_events": sorted(watched_events), + "produced_events": sorted(produced_events), + "flags": sorted(flags), "meta": meta, "config": config, "options_desc": options_desc, "hash": module_hash, "deps": { - "pip": pip_deps, - "pip_constraints": pip_deps_constraints, - "shell": shell_deps, - "apt": apt_deps, + "modules": sorted(deps_modules), + "pip": deps_pip, + "pip_constraints": deps_pip_constraints, + "shell": deps_shell, + "apt": deps_apt, "ansible": ansible_tasks, }, - "sudo": len(apt_deps) > 0, + "sudo": len(deps_apt) > 0, "code": python_code, } if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any( diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 8b803aff4..e96e7320b 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -21,6 +21,8 @@ class BaseModule: flags (List): Flags indicating the type of module (must have at least "safe" or "aggressive" and "passive" or "active"). + deps_modules (List): Other BBOT modules this module depends on. Empty list by default. + deps_pip (List): Python dependencies to install via pip. Empty list by default. deps_apt (List): APT package dependencies to install. Empty list by default. @@ -83,6 +85,7 @@ class BaseModule: options = {} options_desc = {} + deps_modules = [] deps_pip = [] deps_apt = [] deps_shell = [] From 7b8b2b80c9a54fb56f70ab0e86d21faf83d74c84 Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Wed, 13 Mar 2024 15:46:21 -0500 Subject: [PATCH 031/171] Fixed Joel's Spelling Mistakes --- bbot/cli.py | 18 +++ bbot/core/modules.py | 5 +- bbot/scanner/preset/args.py | 17 ++- bbot/scanner/preset/preset.py | 195 +++++++++++++++++++++----- bbot/test/bbot_fixtures.py | 2 +- bbot/test/test_step_1/test_presets.py | 36 ++++- 6 files changed, 229 insertions(+), 44 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 69d056262..449263bc3 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -52,6 +52,24 @@ async def _main(): sys.exit(0) return + # --list-modules + if preset.args.parsed.list_modules: + log.stdout("") + log.stdout("### MODULES ###") + log.stdout("") + for row in preset.module_loader.modules_table(preset.modules).splitlines(): + log.stdout(row) + return + + # --list-flags + if preset.args.parsed.list_flags: + log.stdout("") + log.stdout("### FLAGS ###") + log.stdout("") + for row in preset.module_loader.flags_table(flags=preset.flags).splitlines(): + log.stdout(row) + return + scan = Scanner(preset=preset) await scan.async_start_without_generator() diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 81682f6e8..dc050e24d 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -25,7 +25,10 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") # if a module consumes these event types, automatically assume these dependencies - default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx"} + default_module_deps = { + "HTTP_RESPONSE": "httpx", + "URL": "httpx" + } def __init__(self, preset): self.preset = preset diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 30163ab70..a336a3e89 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -95,21 +95,24 @@ def preset_from_args(self): args_preset.blacklist.add_target(self.parsed.blacklist) args_preset.strict_scope = self.parsed.strict_scope - # modules - args_preset.scan_modules = self.parsed.modules - args_preset.output_modules = self.parsed.output_modules + # modules && flags (excluded then required then all others) args_preset.exclude_modules = self.parsed.exclude_modules + args_preset.exclude_flags = self.parsed.exclude_flags + args_preset.require_flags = self.parsed.require_flags + args_preset.modules = self.parsed.modules - # flags + for output_module in self.parsed.output_modules: + args_preset.add_module(output_module) + args_preset.flags = self.parsed.flags - args_preset.require_flags = self.parsed.require_flags - args_preset.exclude_flags = self.parsed.exclude_flags + + # additional custom presets / config options for preset_param in self.parsed.preset: if Path(preset_param).is_file(): try: - custom_preset = self.preset.from_yaml(preset_param) + custom_preset = self.preset.from_yaml_file(preset_param) except Exception as e: log_to_stderr(f"Error parsing custom config at {preset_param}: {e}", level="ERROR") sys.exit(2) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 4d037a928..15bf7e258 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,4 +1,5 @@ import yaml +import logging import omegaconf from pathlib import Path from contextlib import suppress @@ -8,6 +9,8 @@ from bbot.core.errors import ValidationError +log = logging.getLogger("bbot.presets") + bbot_code_dir = Path(__file__).parent.parent.parent @@ -37,6 +40,13 @@ def __init__( self._helpers = None self._module_loader = None + self._modules = set() + + self._exclude_modules = set() + self._require_flags = set() + self._exclude_flags = set() + self._flags = set() + self._verbose = False self._debug = False self._silent = False @@ -48,6 +58,11 @@ def __init__( # merge any custom configs self.core.merge_custom(config) + # dirs to load modules from + self.module_dirs = self.core.config.get("module_dirs", []) + self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] + self.module_dirs = set(self.module_dirs) + # modules if modules is None: modules = [] @@ -57,8 +72,9 @@ def __init__( modules = [modules] if isinstance(output_modules, str): output_modules = [output_modules] - self.scan_modules = set(modules if modules is not None else []) - self.output_modules = set(output_modules if output_modules is not None else []) + self.modules = set(modules if modules is not None else []) + for output_module in set(output_modules if output_modules is not None else []): + self.add_module(output_module) self.exclude_modules = set(exclude_modules if exclude_modules is not None else []) # module flags @@ -69,11 +85,6 @@ def __init__( # PRESET TODO: preparation of environment # self.core.environ.prepare() - # dirs to load modules from - self.module_dirs = self.core.config.get("module_dirs", []) - self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] - self.module_dirs = set(self.module_dirs) - self.strict_scope = strict_scope # target / whitelist / blacklist @@ -106,14 +117,13 @@ def merge(self, other): if combined_module_dirs != current_module_dirs: self.module_dirs = combined_module_dirs # TODO: refresh module dirs - # modules - self.scan_modules = set(self.scan_modules).union(set(other.scan_modules)) - self.output_modules = set(self.output_modules).union(set(other.output_modules)) + # modules + flags self.exclude_modules = set(self.exclude_modules).union(set(other.exclude_modules)) - # flags - self.flags = set(self.flags).union(set(other.flags)) self.require_flags = set(self.require_flags).union(set(other.require_flags)) self.exclude_flags = set(self.exclude_flags).union(set(other.exclude_flags)) + self.flags = set(self.flags).union(set(other.flags)) + for module_name in other.modules: + self.add_module(module_name) # scope self.target.add_target(other.target) self.whitelist = other.whitelist @@ -146,6 +156,122 @@ def parse_args(self): # validate config / modules / flags # self.args.validate() + @property + def scan_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "scan"] + + @property + def output_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "output"] + + @property + def internal_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] + + @property + def modules(self): + return self._modules + + @modules.setter + def modules(self, modules): + modules = set(modules) + for module_name in modules: + self.add_module(module_name) + + def add_module(self, module_name): + if module_name in self.exclude_modules: + log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + return + try: + preloaded = self.module_loader.preloaded()[module_name] + except KeyError: + raise KeyError(f'Unable to add unknown module "{module_name}": {e}') + + module_flags = preloaded.get("flags", []) + for f in module_flags: + if f in self.exclude_flags: + log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + return + if self.require_flags and f not in self.require_flags: + log.verbose(f'Skipping module "{module_name}" because it doesn\'t have the required flags') + return + + if module_name not in self.modules: + log.verbose(f'Enabling module "{module_name}"') + self.modules.add(module_name) + for module_dep in preloaded.get("deps", {}).get("modules", []): + if module_dep not in self.modules: + log.verbose(f'Enabling module "{module_dep}" because {module_name} depends on it') + self.add_module(module_dep) + + @property + def exclude_modules(self): + return self._exclude_modules + + @property + def exclude_flags(self): + return self._exclude_flags + + @property + def require_flags(self): + return self._require_flags + + @property + def flags(self): + return self._flags + + @flags.setter + def flags(self, flags): + self._flags = set() + for module, preloaded in self.module_loader.preloaded().items(): + module_flags = preloaded.get("flags", []) + if any(f in module_flags for f in module_flags): + self.add_module(module) + + @require_flags.setter + def require_flags(self, flags): + self._require_flags = set() + for flag in flags: + self.add_required_flag(flag) + + @exclude_modules.setter + def exclude_modules(self, modules): + self._exclude_modules = set() + for module in modules: + self.add_excluded_module(module) + + @exclude_flags.setter + def exclude_flags(self, flags): + self._exclude_flags = set() + for flag in flags: + self.add_excluded_flag(flag) + + def add_required_flag(self, flag): + self.require_flags.add(flag) + for module in list(self.modules): + module_flags = self.preloaded_module(module).get("flags", []) + if flag not in module_flags: + log.verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') + self.modules.remove(module) + + def add_excluded_flag(self, flag): + self.exclude_flags.add(flag) + for module in list(self.modules): + module_flags = self.preloaded_module(module).get("flags", []) + if flag in module_flags: + log.verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') + self.modules.remove(module) + + def add_excluded_module(self, module): + self.exclude_modules.add(module) + for module in list(self.modules): + if module in self.exclude_modules: + log.verbose(f'Removing module "{module}" because is excluded') + self.modules.remove(module) + + def preloaded_module(self, module): + return self.module_loader.preloaded()[module] + @property def config(self): return self.core.config @@ -236,10 +362,6 @@ def module_loader(self): def internal_modules(self): return list(self.module_loader.preloaded(type="internal")) - @property - def all_modules(self): - return sorted(self.scan_modules.union(self.output_modules).union(self.internal_modules)) - @property def environ(self): if self._environ is None: @@ -285,13 +407,12 @@ def whitelisted(self, e): """ e = make_event(e, dummy=True) return e in self.whitelist + + + @classmethod - def from_yaml(cls, yaml_preset): - if Path(yaml_preset).is_file(): - preset_dict = omegaconf.OmegaConf.load(yaml_preset) - else: - preset_dict = omegaconf.OmegaConf.create(yaml_preset) + def from_dict(cls, preset_dict): new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -309,6 +430,14 @@ def from_yaml(cls, yaml_preset): strict_scope=preset_dict.get("strict_scope", False), ) return new_preset + + @classmethod + def from_yaml_file(cls, yaml_preset): + return cls.from_dict(omegaconf.OmegaConf.load(yaml_preset)) + + @classmethod + def from_yaml_string(cls, yaml_preset): + return cls.from_dict(omegaconf.OmegaConf.create(yaml_preset)) def to_dict(self, include_target=False, full_config=False): preset_dict = {} @@ -336,21 +465,23 @@ def to_dict(self, include_target=False, full_config=False): if self.strict_scope: preset_dict["strict_scope"] = True - # modules - if self.scan_modules: - preset_dict["modules"] = sorted(self.scan_modules) - if self.output_modules: - preset_dict["output_modules"] = sorted(self.output_modules) - if self.exclude_modules: - preset_dict["exclude_modules"] = sorted(self.exclude_modules) - - # flags - if self.flags: - preset_dict["flags"] = sorted(self.flags) + # flags + modules + # establish requirements / exclusions first if self.require_flags: preset_dict["require_flags"] = sorted(self.require_flags) if self.exclude_flags: preset_dict["exclude_flags"] = sorted(self.exclude_flags) + if self.exclude_modules: + preset_dict["exclude_modules"] = sorted(self.exclude_modules) + # then it's okay to start enabling modules + if self.flags: + preset_dict["flags"] = sorted(self.flags) + scan_modules = self.scan_modules + output_modules = self.output_modules + if scan_modules: + preset_dict["modules"] = sorted(scan_modules) + if output_modules: + preset_dict["output_modules"] = sorted(output_modules) # log verbosity if self.verbose: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 36d9ee889..8cbdd0922 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -227,7 +227,7 @@ def agent(monkeypatch, bbot_config): # bbot config from bbot.scanner import Preset -default_preset = Preset.from_yaml(Path(__file__).parent / "test.conf") +default_preset = Preset.from_yaml_file(Path(__file__).parent / "test.conf") test_config = default_preset.config available_modules = list(default_preset.module_loader.configs(type="scan")) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 27ed0a95d..5ba2858b0 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -74,7 +74,7 @@ def test_preset_yaml(): # test yaml save/load yaml1 = preset1.to_yaml(sort_keys=True) - preset2 = Preset.from_yaml(yaml1) + preset2 = Preset.from_yaml_string(yaml1) yaml2 = preset2.to_yaml(sort_keys=True) assert yaml1 == yaml2 @@ -106,7 +106,7 @@ def test_preset_scope(): # test yaml save/load yaml1 = preset1.to_yaml(sort_keys=True) - preset2 = Preset.from_yaml(yaml1) + preset2 = Preset.from_yaml_string(yaml1) yaml2 = preset2.to_yaml(sort_keys=True) assert yaml1 == yaml2 @@ -143,7 +143,7 @@ def test_preset_scope(): assert not preset1.in_scope("asdf.test.www.evilcorp.ce") -def test_preset_misc(): +def test_preset_logging(): # test verbosity levels (conflicting verbose/debug/silent) preset = Preset(verbose=True) assert preset.verbose == True @@ -154,6 +154,36 @@ def test_preset_misc(): assert preset.verbose == False assert preset.debug == True assert preset.silent == False + assert preset.core.logger.log_level == logging.DEBUG + preset.silent = True + assert preset.verbose == False + assert preset.debug == False + assert preset.silent == True + assert preset.core.logger.log_level == logging.CRITICAL + + +def test_preset_module_resolution(): + + # make sure module dependency resolution works as expected + preset = Preset() + assert not preset.scan_modules + preset.scan_modules = ["wappalyzer"] + assert preset.scan_modules == {"wappalyzer", "httpx"} + + # make sure flags work as expected + preset = Preset() + assert not preset.flags + assert not preset.scan_modules + preset.require_flags = ["safe"] + preset.exclude_flags = ["slow"] + preset.exclude_modules = ["c99"] + preset.flags = ["subdomain-enum"] + + assert preset.scan_modules + # test exclude_modules + # test exclude_flags + # test require_flags + # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation From 942f3f7b3d085e0a59b4e8ae2fef8fb878c942bf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 14 Mar 2024 17:12:17 -0400 Subject: [PATCH 032/171] steady work on presets - module loading + more tests --- bbot/core/modules.py | 155 ++++++++++++-------- bbot/scanner/preset/args.py | 15 +- bbot/scanner/preset/preset.py | 194 ++++++++++++++++---------- bbot/scanner/scanner.py | 4 +- bbot/test/test_step_1/test_presets.py | 149 ++++++++++++++++++-- 5 files changed, 358 insertions(+), 159 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index dc050e24d..1e8bf70db 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -2,17 +2,25 @@ import ast import sys import pickle +import logging import importlib import traceback from pathlib import Path from omegaconf import OmegaConf from contextlib import suppress +from bbot.core import CORE + from .flags import flag_descriptions from .helpers.logger import log_to_stderr from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform +log = logging.getLogger("bbot.module_loader") + +bbot_code_dir = Path(__file__).parent.parent + + class ModuleLoader: """ Main class responsible for preloading BBOT modules. @@ -22,26 +30,42 @@ class ModuleLoader: This ensures that all requisite libraries and components are available for the module to function correctly. """ + default_module_dir = bbot_code_dir / "modules" + module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") # if a module consumes these event types, automatically assume these dependencies - default_module_deps = { - "HTTP_RESPONSE": "httpx", - "URL": "httpx" - } - - def __init__(self, preset): - self.preset = preset - self.__preloaded = None + default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx"} + + def __init__(self): + self.__preloaded = {} self._preloaded_orig = None self._modules = {} self._configs = {} - self.preload_cache_file = self.preset.core.cache_dir / "preloaded" + self.preload_cache_file = CORE.cache_dir / "preloaded" self._preload_cache = None - # expand to include all recursive dirs - self.module_dirs = self.get_recursive_dirs(*self.preset.module_dirs) + self._module_dirs = set() + self._module_dirs_preloaded = set() + self.add_module_dir(self.default_module_dir) + + @property + def module_dirs(self): + return self._module_dirs + + def add_module_dir(self, module_dir): + module_dir = Path(module_dir).resolve() + if not module_dir.is_dir(): + log.warning(f'Failed to add custom module dir "{module_dir}", please make sure it exists') + return + new_module_dirs = set() + for _module_dir in self.get_recursive_dirs(module_dir): + _module_dir = Path(_module_dir).resolve() + if _module_dir not in self._module_dirs: + self._module_dirs.add(_module_dir) + new_module_dirs.add(_module_dir) + self.preload(module_dirs=new_module_dirs) def file_filter(self, file): file = file.resolve() @@ -49,7 +73,7 @@ def file_filter(self, file): return False return file.suffix.lower() == ".py" and file.stem not in ["base", "__init__"] - def preload(self): + def preload(self, module_dirs=None): """Preloads all BBOT modules. This function recursively iterates through each file in the module directories @@ -70,47 +94,58 @@ def preload(self): ... } """ - if self.__preloaded is None: - self.__preloaded = {} - for module_dir in self.module_dirs: - for module_file in list_files(module_dir, filter=self.file_filter): - module_name = module_file.stem - module_file = module_file.resolve() - - # try to load from cache - module_cache_key = (str(module_file), tuple(module_file.stat())) - cache_key = self.preload_cache.get(module_name, {}).get("cache_key", ()) - if module_cache_key == cache_key: - preloaded = self.preload_cache[module_name] - else: - if module_dir.name == "modules": - namespace = f"bbot.modules" - else: - namespace = f"bbot.modules.{module_dir.name}" - try: - preloaded = self.preload_module(module_file) - module_type = "scan" - if module_dir.name in ("output", "internal"): - module_type = str(module_dir.name) - elif module_dir.name not in ("modules"): - preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) - preloaded["type"] = module_type - preloaded["namespace"] = namespace - preloaded["cache_key"] = module_cache_key - - except Exception: - log_to_stderr( - f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL" - ) - log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") - sys.exit(1) - - self.__preloaded[module_name] = preloaded - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_name] = config - - self.preload_cache = self.__preloaded + if module_dirs is None: + module_dirs = self.module_dirs + for module_dir in module_dirs: + if module_dir in self._module_dirs_preloaded: + log.debug(f'Custom module dir "{module_dir}" was already added') + continue + + for module_file in list_files(module_dir, filter=self.file_filter): + module_name = module_file.stem + module_file = module_file.resolve() + + # try to load from cache + module_cache_key = (str(module_file), tuple(module_file.stat())) + cache_key = self.preload_cache.get(module_name, {}).get("cache_key", ()) + if module_cache_key == cache_key: + preloaded = self.preload_cache[module_name] + else: + if module_dir.name == "modules": + namespace = f"bbot.modules" + else: + namespace = f"bbot.modules.{module_dir.name}" + try: + preloaded = self.preload_module(module_file) + module_type = "scan" + if module_dir.name in ("output", "internal"): + module_type = str(module_dir.name) + elif module_dir.name not in ("modules"): + preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) + # derive module dependencies from watched event types (only for scan modules) + if module_type == "scan": + for event_type in preloaded["watched_events"]: + if event_type in self.default_module_deps: + deps_modules = set(preloaded.get("deps", {}).get("modules", [])) + deps_modules.add(self.default_module_deps[event_type]) + preloaded["deps"]["modules"] = deps_modules + preloaded["type"] = module_type + preloaded["namespace"] = namespace + preloaded["cache_key"] = module_cache_key + + except Exception: + log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") + log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") + sys.exit(1) + + self.__preloaded[module_name] = preloaded + config = OmegaConf.create(preloaded.get("config", {})) + self._configs[module_name] = config + + self._module_dirs_preloaded.add(module_dir) + + self.save_preload_cache() return self.__preloaded @property @@ -129,12 +164,15 @@ def preload_cache(self, value): with open(self.preload_cache_file, "wb") as f: pickle.dump(self._preload_cache, f) + def save_preload_cache(self): + self.preload_cache = self.__preloaded + @property def _preloaded(self): - return self.preload() + return self.__preloaded def get_recursive_dirs(self, *dirs): - dirs = set(Path(d) for d in dirs) + dirs = set(Path(d).resolve() for d in dirs) for d in list(dirs): if not d.is_dir(): continue @@ -152,7 +190,6 @@ def preloaded(self, type=None): return preloaded def configs(self, type=None): - self.preload() configs = {} if type is not None: configs = {k: v for k, v in self._configs.items() if self.check_type(k, type)} @@ -302,11 +339,6 @@ def preload_module(self, module_file): elif os_platform() == "darwin" and ("package" in task and task.get("become", False) == True): task["become"] = False - # derive module dependencies from watched event types - for event_type in watched_events: - if event_type in self.default_module_deps: - deps_modules.add(self.default_module_deps[event_type]) - preloaded_data = { "watched_events": sorted(watched_events), "produced_events": sorted(produced_events), @@ -581,3 +613,6 @@ def filter_modules(self, modules=None, mod_type=None): module_list.sort(key=lambda x: "passive" in x[-1]["flags"]) module_list.sort(key=lambda x: x[-1]["type"], reverse=True) return module_list + + +module_loader = ModuleLoader() diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index a336a3e89..b6ae260b7 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -103,10 +103,8 @@ def preset_from_args(self): for output_module in self.parsed.output_modules: args_preset.add_module(output_module) - - args_preset.flags = self.parsed.flags - + args_preset.flags = self.parsed.flags # additional custom presets / config options for preset_param in self.parsed.preset: @@ -126,9 +124,12 @@ def preset_from_args(self): args_preset.core.merge_custom(cli_config) # verbosity levels - args_preset.silent = self.parsed.silent - args_preset.verbose = self.parsed.verbose - args_preset.debug = self.parsed.debug + if self.parsed.silent: + args_preset.silent = self.parsed.silent + if self.parsed.verbose: + args_preset.verbose = self.parsed.verbose + if self.parsed.debug: + args_preset.debug = self.parsed.debug return args_preset def create_parser(self, *args, **kwargs): @@ -198,7 +199,7 @@ def create_parser(self, *args, **kwargs): "-om", "--output-modules", nargs="+", - default=["human", "json", "csv"], + default=[], help=f'Output module(s). Choices: {",".join(self._output_module_choices)}', metavar="MODULE", ) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 15bf7e258..23c188a9b 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -11,13 +11,9 @@ log = logging.getLogger("bbot.presets") -bbot_code_dir = Path(__file__).parent.parent.parent - class Preset: - default_module_dir = bbot_code_dir / "modules" - def __init__( self, *targets, @@ -26,6 +22,7 @@ def __init__( modules=None, output_modules=None, exclude_modules=None, + internal_modules=None, flags=None, require_flags=None, exclude_flags=None, @@ -34,6 +31,7 @@ def __init__( silent=False, config=None, strict_scope=False, + module_dirs=None, ): self._args = None self._environ = None @@ -51,6 +49,12 @@ def __init__( self._debug = False self._silent = False + # custom module directories + self.module_dirs = module_dirs + if self.module_dirs is not None: + for m in self.module_dirs: + self.module_loader.add_module_dir(m) + # bbot core config self.core = CORE.copy() if config is None: @@ -58,29 +62,24 @@ def __init__( # merge any custom configs self.core.merge_custom(config) - # dirs to load modules from - self.module_dirs = self.core.config.get("module_dirs", []) - self.module_dirs = [Path(p) for p in self.module_dirs] + [self.default_module_dir] - self.module_dirs = set(self.module_dirs) - - # modules + # modules + flags if modules is None: modules = [] if output_modules is None: - output_modules = ["python"] + output_modules = ["python", "csv", "human", "json"] + if internal_modules is None: + internal_modules = ["aggregate", "excavate", "speculate"] if isinstance(modules, str): modules = [modules] if isinstance(output_modules, str): output_modules = [output_modules] - self.modules = set(modules if modules is not None else []) - for output_module in set(output_modules if output_modules is not None else []): - self.add_module(output_module) - self.exclude_modules = set(exclude_modules if exclude_modules is not None else []) - - # module flags - self.flags = set(flags if flags is not None else []) - self.require_flags = set(require_flags if require_flags is not None else []) - self.exclude_flags = set(exclude_flags if exclude_flags is not None else []) + self.exclude_modules = exclude_modules if exclude_modules is not None else [] + self.require_flags = require_flags if require_flags is not None else [] + self.exclude_flags = exclude_flags if exclude_flags is not None else [] + self.flags = flags if flags is not None else [] + self.scan_modules = modules if modules is not None else [] + self.output_modules = output_modules if output_modules is not None else [] + self.internal_modules = internal_modules if internal_modules is not None else [] # PRESET TODO: preparation of environment # self.core.environ.prepare() @@ -111,19 +110,19 @@ def __init__( def merge(self, other): # module dirs - current_module_dirs = set(self.module_dirs) - other_module_dirs = set(other.module_dirs) - combined_module_dirs = current_module_dirs.union(other_module_dirs) - if combined_module_dirs != current_module_dirs: - self.module_dirs = combined_module_dirs - # TODO: refresh module dirs + if other.module_dirs: + self.refresh_module_loader() + # TODO: find-and-replace module configs # modules + flags + # establish requirements / exclusions first self.exclude_modules = set(self.exclude_modules).union(set(other.exclude_modules)) self.require_flags = set(self.require_flags).union(set(other.require_flags)) self.exclude_flags = set(self.exclude_flags).union(set(other.exclude_flags)) + # then it's okay to start enabling modules self.flags = set(self.flags).union(set(other.flags)) for module_name in other.modules: - self.add_module(module_name) + module_type = self.preloaded_module(module_name).get("type", "scan") + self.add_module(module_name, module_type=module_type) # scope self.target.add_target(other.target) self.whitelist = other.whitelist @@ -134,9 +133,12 @@ def merge(self, other): # config self.core.merge_custom(other.core.custom_config) # log verbosity - self.silent = other.silent - self.verbose = other.verbose - self.debug = other.debug + if other.silent: + self.silent = other.silent + if other.verbose: + self.verbose = other.verbose + if other.debug: + self.debug = other.debug def parse_args(self): @@ -156,6 +158,10 @@ def parse_args(self): # validate config / modules / flags # self.args.validate() + @property + def modules(self): + return self._modules + @property def scan_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "scan"] @@ -168,33 +174,63 @@ def output_modules(self): def internal_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] - @property - def modules(self): - return self._modules - @modules.setter def modules(self, modules): + if isinstance(modules, str): + modules = [modules] modules = set(modules) for module_name in modules: self.add_module(module_name) - def add_module(self, module_name): + @scan_modules.setter + def scan_modules(self, modules): + self._modules_setter(modules, module_type="scan") + + @output_modules.setter + def output_modules(self, modules): + self._modules_setter(modules, module_type="output") + + @internal_modules.setter + def internal_modules(self, modules): + self._modules_setter(modules, module_type="internal") + + def _modules_setter(self, modules, module_type="scan"): + if isinstance(modules, str): + modules = [modules] + # start by removing currently-enabled modules of that type + for module_name in list(self.modules): + if module_type and self.preloaded_module(module_name).get("type", "scan") == module_type: + self._modules.remove(module_name) + for module_name in set(modules): + self.add_module(module_name, module_type=module_type) + + def add_module(self, module_name, module_type="scan"): if module_name in self.exclude_modules: log.verbose(f'Skipping module "{module_name}" because it\'s excluded') return try: preloaded = self.module_loader.preloaded()[module_name] except KeyError: - raise KeyError(f'Unable to add unknown module "{module_name}": {e}') + raise KeyError(f'Unable to add unknown BBOT module "{module_name}"') module_flags = preloaded.get("flags", []) + if module_type: + _module_type = preloaded.get("type", "scan") + if _module_type != module_type: + log.verbose( + f'Not enabling module "{module_name}" because its type ({_module_type}) is not "{module_type}"' + ) + return + for f in module_flags: if f in self.exclude_flags: log.verbose(f'Skipping module "{module_name}" because it\'s excluded') return - if self.require_flags and f not in self.require_flags: - log.verbose(f'Skipping module "{module_name}" because it doesn\'t have the required flags') - return + if self.require_flags and not any(f in self.require_flags for f in module_flags): + log.verbose( + f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' + ) + return if module_name not in self.modules: log.verbose(f'Enabling module "{module_name}"') @@ -222,28 +258,37 @@ def flags(self): @flags.setter def flags(self, flags): - self._flags = set() - for module, preloaded in self.module_loader.preloaded().items(): - module_flags = preloaded.get("flags", []) - if any(f in module_flags for f in module_flags): - self.add_module(module) - - @require_flags.setter + if isinstance(flags, str): + flags = [flags] + self._flags = set(flags) + if self._flags: + for module, preloaded in self.module_loader.preloaded().items(): + module_flags = preloaded.get("flags", []) + if any(f in self._flags for f in module_flags): + self.add_module(module) + + @require_flags.setter def require_flags(self, flags): + if isinstance(flags, str): + flags = [flags] self._require_flags = set() - for flag in flags: + for flag in set(flags): self.add_required_flag(flag) @exclude_modules.setter def exclude_modules(self, modules): + if isinstance(modules, str): + modules = [modules] self._exclude_modules = set() - for module in modules: + for module in set(modules): self.add_excluded_module(module) @exclude_flags.setter def exclude_flags(self, flags): + if isinstance(flags, str): + flags = [flags] self._exclude_flags = set() - for flag in flags: + for flag in set(flags): self.add_excluded_flag(flag) def add_required_flag(self, flag): @@ -266,7 +311,7 @@ def add_excluded_module(self, module): self.exclude_modules.add(module) for module in list(self.modules): if module in self.exclude_modules: - log.verbose(f'Removing module "{module}" because is excluded') + log.verbose(f'Removing module "{module}" because it\'s excluded') self.modules.remove(module) def preloaded_module(self, module): @@ -291,8 +336,8 @@ def silent(self): @verbose.setter def verbose(self, value): if value: - self.debug = False - self.silent = False + self._debug = False + self._silent = False self.core.merge_custom({"verbose": True}) self.core.logger.log_level = "VERBOSE" else: @@ -304,8 +349,8 @@ def verbose(self, value): @debug.setter def debug(self, value): if value: - self.verbose = False - self.silent = False + self._verbose = False + self._silent = False self.core.merge_custom({"debug": True}) self.core.logger.log_level = "DEBUG" else: @@ -317,8 +362,8 @@ def debug(self, value): @silent.setter def silent(self, value): if value: - self.verbose = False - self.debug = False + self._verbose = False + self._debug = False self.core.merge_custom({"silent": True}) self.core.logger.log_level = "CRITICAL" else: @@ -342,25 +387,24 @@ def module_loader(self): # PRESET TODO self.environ if self._module_loader is None: - from bbot.core.modules import ModuleLoader - - self._module_loader = ModuleLoader(self) + from bbot.core.modules import module_loader - # update default config with module defaults - module_config = omegaconf.OmegaConf.create( - { - "modules": self._module_loader.configs(type="scan"), - "output_modules": self._module_loader.configs(type="output"), - "internal_modules": self._module_loader.configs(type="internal"), - } - ) - self.core.merge_default(module_config) + self._module_loader = module_loader + self.refresh_module_loader() return self._module_loader - @property - def internal_modules(self): - return list(self.module_loader.preloaded(type="internal")) + def refresh_module_loader(self): + self.module_loader.preload() + # update default config with module defaults + module_config = omegaconf.OmegaConf.create( + { + "modules": self.module_loader.configs(type="scan"), + "output_modules": self.module_loader.configs(type="output"), + "internal_modules": self.module_loader.configs(type="internal"), + } + ) + self.core.merge_default(module_config) @property def environ(self): @@ -407,9 +451,6 @@ def whitelisted(self, e): """ e = make_event(e, dummy=True) return e in self.whitelist - - - @classmethod def from_dict(cls, preset_dict): @@ -428,9 +469,10 @@ def from_dict(cls, preset_dict): silent=preset_dict.get("silent", False), config=preset_dict.get("config"), strict_scope=preset_dict.get("strict_scope", False), + module_dirs=preset_dict.get("module_dirs", []), ) return new_preset - + @classmethod def from_yaml_file(cls, yaml_preset): return cls.from_dict(omegaconf.OmegaConf.load(yaml_preset)) @@ -466,14 +508,12 @@ def to_dict(self, include_target=False, full_config=False): preset_dict["strict_scope"] = True # flags + modules - # establish requirements / exclusions first if self.require_flags: preset_dict["require_flags"] = sorted(self.require_flags) if self.exclude_flags: preset_dict["exclude_flags"] = sorted(self.exclude_flags) if self.exclude_modules: preset_dict["exclude_modules"] = sorted(self.exclude_modules) - # then it's okay to start enabling modules if self.flags: preset_dict["flags"] = sorted(self.flags) scan_modules = self.scan_modules diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index ff10fc85f..423af6545 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -455,7 +455,7 @@ async def load_modules(self): After all modules are loaded, they are sorted by `_priority` and stored in the `modules` dictionary. """ if not self._modules_loaded: - if not self.preset.all_modules: + if not self.preset.modules: self.warning(f"No modules to load") return @@ -463,7 +463,7 @@ async def load_modules(self): self.warning(f"No scan modules to load") # install module dependencies - succeeded, failed = await self.helpers.depsinstaller.install(*self.preset.all_modules) + succeeded, failed = await self.helpers.depsinstaller.install(*self.preset.modules) if failed: msg = f"Failed to install dependencies for {len(failed):,} modules: {','.join(failed)}" self._fail_setup(msg) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 5ba2858b0..0c174fce1 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -142,6 +142,11 @@ def test_preset_scope(): assert not preset1.in_scope("evilcorp.com") assert not preset1.in_scope("asdf.test.www.evilcorp.ce") + preset4 = Preset(output_modules="neo4j") + set(preset1.output_modules) == {"python", "csv", "human", "json"} + preset1.merge(preset4) + set(preset1.output_modules) == {"python", "csv", "human", "json", "neo4j"} + def test_preset_logging(): # test verbosity levels (conflicting verbose/debug/silent) @@ -163,27 +168,145 @@ def test_preset_logging(): def test_preset_module_resolution(): - - # make sure module dependency resolution works as expected preset = Preset() + sslcert_preloaded = preset.preloaded_module("sslcert") + wayback_preloaded = preset.preloaded_module("wayback") + wappalyzer_preloaded = preset.preloaded_module("wappalyzer") + sslcert_flags = sslcert_preloaded.get("flags", []) + wayback_flags = wayback_preloaded.get("flags", []) + wappalyzer_flags = wappalyzer_preloaded.get("flags", []) + assert "active" in sslcert_flags + assert "passive" in wayback_flags + assert "active" in wappalyzer_flags + assert "subdomain-enum" in sslcert_flags + assert "subdomain-enum" in wayback_flags + assert "httpx" in wappalyzer_preloaded["deps"]["modules"] + + # make sure we have the expected defaults assert not preset.scan_modules - preset.scan_modules = ["wappalyzer"] - assert preset.scan_modules == {"wappalyzer", "httpx"} + assert set(preset.output_modules) == {"python", "csv", "human", "json"} + assert set(preset.internal_modules) == {"aggregate", "excavate", "speculate"} + assert preset.modules == set(preset.output_modules).union(set(preset.internal_modules)) + + # make sure dependency resolution works as expected + preset.modules = ["wappalyzer"] + assert set(preset.scan_modules) == {"wappalyzer", "httpx"} # make sure flags work as expected preset = Preset() assert not preset.flags assert not preset.scan_modules - preset.require_flags = ["safe"] - preset.exclude_flags = ["slow"] - preset.exclude_modules = ["c99"] preset.flags = ["subdomain-enum"] + assert "sslcert" in preset.modules + assert "wayback" in preset.modules + assert "sslcert" in preset.scan_modules + assert "wayback" in preset.scan_modules + + # make sure module exclusions work as expected + preset.exclude_modules = ["sslcert"] + assert "sslcert" not in preset.modules + assert "wayback" in preset.modules + assert "sslcert" not in preset.scan_modules + assert "wayback" in preset.scan_modules + preset.scan_modules = ["sslcert"] + assert "sslcert" not in preset.modules + assert "wayback" not in preset.modules + assert "sslcert" not in preset.scan_modules + assert "wayback" not in preset.scan_modules + preset.exclude_modules = [] + preset.scan_modules = ["sslcert"] + assert "sslcert" in preset.modules + assert "wayback" not in preset.modules + assert "sslcert" in preset.scan_modules + assert "wayback" not in preset.scan_modules + preset.add_module("wayback") + assert "sslcert" in preset.modules + assert "wayback" in preset.modules + assert "sslcert" in preset.scan_modules + assert "wayback" in preset.scan_modules + preset.exclude_modules = ["sslcert"] + assert "sslcert" not in preset.modules + assert "wayback" in preset.modules + assert "sslcert" not in preset.scan_modules + assert "wayback" in preset.scan_modules + + # make sure flag requirements work as expected + preset = Preset() + preset.require_flags = ["passive"] + preset.scan_modules = ["sslcert"] + assert not preset.scan_modules + preset.scan_modules = ["wappalyzer"] + assert not preset.scan_modules + preset.flags = ["subdomain-enum"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + preset.require_flags = [] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + assert not preset.require_flags + preset.flags = [] + preset.scan_modules = [] + assert not preset.flags + assert not preset.scan_modules + preset.scan_modules = ["sslcert", "wayback"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" in preset.modules + assert "sslcert" in preset.scan_modules + preset.require_flags = ["passive"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + + # make sure flag exclusions work as expected + preset = Preset() + preset.exclude_flags = ["active"] + preset.scan_modules = ["sslcert"] + assert not preset.scan_modules + preset.scan_modules = ["wappalyzer"] + assert not preset.scan_modules + preset.flags = ["subdomain-enum"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + preset.exclude_flags = [] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + assert not preset.require_flags + preset.flags = [] + preset.scan_modules = [] + assert not preset.flags + assert not preset.scan_modules + preset.scan_modules = ["sslcert", "wayback"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" in preset.modules + assert "sslcert" in preset.scan_modules + preset.exclude_flags = ["active"] + assert "wayback" in preset.modules + assert "wayback" in preset.scan_modules + assert "sslcert" not in preset.modules + assert "sslcert" not in preset.scan_modules + + +def test_preset_module_loader(): + # preset = Preset() + # ensure custom module dir works + # ensure default configs are refreshed + # ensure find-and-replace happens + # ensure + - assert preset.scan_modules - # test exclude_modules - # test exclude_flags - # test require_flags +# test recursive include - # test custom module load directory - # make sure it works with cli arg module/flag/config syntax validation +# test custom module load directory +# make sure it works with cli arg module/flag/config syntax validation From 76a83ce25e788921accd28c663e574004ff396df Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Fri, 15 Mar 2024 12:20:48 -0500 Subject: [PATCH 033/171] Added module_type to add_module call for all Output Modules --- bbot/scanner/preset/args.py | 2 +- bbot/scanner/preset/preset.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index b6ae260b7..ff0cf5ed2 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -102,7 +102,7 @@ def preset_from_args(self): args_preset.modules = self.parsed.modules for output_module in self.parsed.output_modules: - args_preset.add_module(output_module) + args_preset.add_module(output_module, module_type="output") args_preset.flags = self.parsed.flags diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 23c188a9b..55817002f 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -205,6 +205,7 @@ def _modules_setter(self, modules, module_type="scan"): self.add_module(module_name, module_type=module_type) def add_module(self, module_name, module_type="scan"): + # log.info(f'Adding "{module_name}": {module_type}') if module_name in self.exclude_modules: log.verbose(f'Skipping module "{module_name}" because it\'s excluded') return From 90455155698d03e2d385fcd1b53ffe3914fcdfd6 Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Fri, 15 Mar 2024 12:51:54 -0500 Subject: [PATCH 034/171] Added Debug Statement in Installer.py --- bbot/core/helpers/depsinstaller/installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index d97fb577d..c5fe716af 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -71,6 +71,7 @@ async def install(self, *modules): failed.append(m) continue preloaded = self.all_modules_preloaded[m] + log.debug(f"Installing {m} - Preloaded Deps {preloaded['deps']}") # make a hash of the dependencies and check if it's already been handled # take into consideration whether the venv or bbot home directory changes module_hash = self.parent_helper.sha1( From edd241841f94f5f2dce73be292105f71c5bfb5aa Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 15 Mar 2024 14:28:42 -0400 Subject: [PATCH 035/171] steady work on presets - module loader tests --- bbot/core/core.py | 20 ++++- bbot/core/modules.py | 15 +++- bbot/scanner/preset/preset.py | 26 ++++-- bbot/test/test_step_1/test_presets.py | 111 ++++++++++++++++++++++++-- 4 files changed, 155 insertions(+), 17 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index f87567b32..bb509c0ad 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -18,10 +18,6 @@ def __init__(self): # ensure bbot home dir if not "home" in self.config: self.custom_config["home"] = "~/.bbot" - self.home = Path(self.config["home"]).expanduser().resolve() - self.cache_dir = self.home / "cache" - self.tools_dir = self.home / "tools" - self.scans_dir = self.home / "scans" # bare minimum == logging self.logger @@ -30,6 +26,22 @@ def __init__(self): # - check_cli_args # - ensure_config_files + @property + def home(self): + return Path(self.config["home"]).expanduser().resolve() + + @property + def cache_dir(self): + return self.home / "cache" + + @property + def tools_dir(self): + return self.home / "tools" + + @property + def scans_dir(self): + return self.home / "scans" + @property def config(self): """ diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 1e8bf70db..f3b980284 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -1,6 +1,7 @@ import re import ast import sys +import atexit import pickle import logging import importlib @@ -38,24 +39,35 @@ class ModuleLoader: default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx"} def __init__(self): + self.core = CORE + self.__preloaded = {} self._preloaded_orig = None self._modules = {} self._configs = {} - self.preload_cache_file = CORE.cache_dir / "preloaded" self._preload_cache = None self._module_dirs = set() self._module_dirs_preloaded = set() self.add_module_dir(self.default_module_dir) + # save preload cache before exiting + atexit.register(self.save_preload_cache) + + @property + def preload_cache_file(self): + return self.core.cache_dir / "module_preload_cache" + @property def module_dirs(self): return self._module_dirs def add_module_dir(self, module_dir): module_dir = Path(module_dir).resolve() + if module_dir in self._module_dirs: + log.debug(f'Already added custom module dir "{module_dir}"') + return if not module_dir.is_dir(): log.warning(f'Failed to add custom module dir "{module_dir}", please make sure it exists') return @@ -145,7 +157,6 @@ def preload(self, module_dirs=None): self._module_dirs_preloaded.add(module_dir) - self.save_preload_cache() return self.__preloaded @property diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 55817002f..77b0ce343 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -50,10 +50,8 @@ def __init__( self._silent = False # custom module directories + self._module_dirs = set() self.module_dirs = module_dirs - if self.module_dirs is not None: - for m in self.module_dirs: - self.module_loader.add_module_dir(m) # bbot core config self.core = CORE.copy() @@ -106,9 +104,14 @@ def __init__( if silent: self.silent = silent - self.bbot_home = Path(self.config.get("home", "~/.bbot")).expanduser().resolve() + @property + def bbot_home(self): + return Path(self.config.get("home", "~/.bbot")).expanduser().resolve() def merge(self, other): + # config + self.core.merge_custom(other.core.custom_config) + self.module_loader.core = self.core # module dirs if other.module_dirs: self.refresh_module_loader() @@ -130,8 +133,6 @@ def merge(self, other): self.strict_scope = self.strict_scope or other.strict_scope for t in (self.target, self.whitelist): t.strict_scope = self.strict_scope - # config - self.core.merge_custom(other.core.custom_config) # log verbosity if other.silent: self.silent = other.silent @@ -158,6 +159,19 @@ def parse_args(self): # validate config / modules / flags # self.args.validate() + @property + def module_dirs(self): + return self.module_loader.module_dirs + + @module_dirs.setter + def module_dirs(self, module_dirs): + if module_dirs: + if isinstance(module_dirs, str): + module_dirs = [module_dirs] + for m in module_dirs: + self.module_loader.add_module_dir(m) + self._module_dirs.add(m) + @property def modules(self): return self._modules diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 0c174fce1..5ebe9a7ec 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -298,11 +298,112 @@ def test_preset_module_resolution(): def test_preset_module_loader(): - # preset = Preset() - # ensure custom module dir works - # ensure default configs are refreshed - # ensure find-and-replace happens - # ensure + + from pathlib import Path + + custom_module_dir = Path("/tmp/.bbot_test/custom_module_dir") + custom_module_dir_2 = custom_module_dir / "asdf" + custom_output_module_dir = custom_module_dir / "output" + custom_internal_module_dir = custom_module_dir / "internal" + for d in [custom_module_dir, custom_module_dir_2, custom_output_module_dir, custom_internal_module_dir]: + d.mkdir(parents=True, exist_ok=True) + assert d.is_dir() + custom_module_1 = custom_module_dir / "testmodule1.py" + with open(custom_module_1, "w") as f: + f.write( + """ +from bbot.modules.base import BaseModule + +class TestModule1(BaseModule): + watched_events = ["URL", "HTTP_RESPONSE"] + produced_events = ["VULNERABILITY"] +""" + ) + + custom_module_2 = custom_output_module_dir / "testmodule2.py" + with open(custom_module_2, "w") as f: + f.write( + """ +from bbot.modules.output.base import BaseOutputModule + +class TestModule2(BaseOutputModule): + pass +""" + ) + + custom_module_3 = custom_internal_module_dir / "testmodule3.py" + with open(custom_module_3, "w") as f: + f.write( + """ +from bbot.modules.internal.base import BaseInternalModule + +class TestModule3(BaseInternalModule): + pass +""" + ) + + custom_module_4 = custom_module_dir_2 / "testmodule4.py" + with open(custom_module_4, "w") as f: + f.write( + """ +from bbot.modules.base import BaseModule + +class TestModule4(BaseModule): + watched_events = ["TECHNOLOGY"] + produced_events = ["FINDING"] +""" + ) + + assert custom_module_1.is_file() + assert custom_module_2.is_file() + assert custom_module_3.is_file() + assert custom_module_4.is_file() + + preset = Preset() + preset.module_loader.save_preload_cache() + assert preset.module_loader.preload_cache_file.is_file() + + # at this point, core modules should be loaded, but not custom ones + assert "wappalyzer" in preset.module_loader.preloaded() + assert "testmodule1" not in preset.module_loader.preloaded() + + import pickle + + with open(preset.module_loader.preload_cache_file, "rb") as f: + preloaded = pickle.load(f) + assert "wappalyzer" in preloaded + assert "testmodule1" not in preloaded + + # add custom module dir + preset.module_dirs = [str(custom_module_dir)] + assert custom_module_dir in preset.module_dirs + assert custom_module_dir_2 in preset.module_dirs + assert custom_output_module_dir in preset.module_dirs + assert custom_internal_module_dir in preset.module_dirs + + # now our custom modules should be loaded + assert "wappalyzer" in preset.module_loader.preloaded() + assert "testmodule1" in preset.module_loader.preloaded() + assert "testmodule2" in preset.module_loader.preloaded() + assert "testmodule3" in preset.module_loader.preloaded() + assert "testmodule4" in preset.module_loader.preloaded() + + preset.module_loader.save_preload_cache() + with open(preset.module_loader.preload_cache_file, "rb") as f: + preloaded = pickle.load(f) + assert "wappalyzer" in preloaded + assert "testmodule1" in preloaded + assert "testmodule2" in preloaded + assert "testmodule3" in preloaded + assert "testmodule4" in preloaded + + # since module loader is shared across all presets, a new preset should now also have our custom modules + preset2 = Preset() + assert "wappalyzer" in preset2.module_loader.preloaded() + assert "testmodule1" in preset2.module_loader.preloaded() + assert "testmodule2" in preset2.module_loader.preloaded() + assert "testmodule3" in preset2.module_loader.preloaded() + assert "testmodule4" in preset2.module_loader.preloaded() # test recursive include From ed4083dd79b24416bfa9c2aa30ecf815d46f117e Mon Sep 17 00:00:00 2001 From: Jack Ward Date: Fri, 15 Mar 2024 15:02:01 -0500 Subject: [PATCH 036/171] Added launch.json for Live Debugging --- .vscode/launch.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..2c3f13080 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "version": "0.1.0", + "configurations": [ + { + "name": "bbot", + "type": "python", + "request": "launch", + "cwd": "${workspaceFolder}", + "module": "bbot.cli", + "args": ["-m httpx", "-t example.com"] + } + ] +} \ No newline at end of file From 67b84a9f48ce354dd52c4144372947053d895af4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 15 Mar 2024 16:10:16 -0400 Subject: [PATCH 037/171] fix deps error --- bbot/core/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index f3b980284..e121313ac 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -141,7 +141,7 @@ def preload(self, module_dirs=None): if event_type in self.default_module_deps: deps_modules = set(preloaded.get("deps", {}).get("modules", [])) deps_modules.add(self.default_module_deps[event_type]) - preloaded["deps"]["modules"] = deps_modules + preloaded["deps"]["modules"] = sorted(deps_modules) preloaded["type"] = module_type preloaded["namespace"] = namespace preloaded["cache_key"] = module_cache_key From 1ad6a58825048a89467751a145407181f8a92e6e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 15 Mar 2024 17:55:51 -0400 Subject: [PATCH 038/171] don't cache module python code --- bbot/core/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index e121313ac..42b1ce862 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -367,7 +367,6 @@ def preload_module(self, module_file): "ansible": ansible_tasks, }, "sudo": len(deps_apt) > 0, - "code": python_code, } if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any( x == True for x in search_dict_by_key("ansible_become", ansible_tasks) From 94170252db0a077d1e20ef7f0bf26669714a25f6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 17 Mar 2024 18:53:31 -0400 Subject: [PATCH 039/171] fixed preload find-and-replace --- bbot/agent/agent.py | 2 +- bbot/cli.py | 2 + bbot/core/config/files.py | 2 +- bbot/core/core.py | 8 ++ bbot/core/helpers/depsinstaller/installer.py | 17 ++-- bbot/core/helpers/helper.py | 2 +- bbot/core/helpers/web.py | 6 +- bbot/core/modules.py | 17 ++-- bbot/defaults.yml | 8 ++ bbot/modules/internal/base.py | 7 -- bbot/modules/output/base.py | 7 -- bbot/scanner/preset/args.py | 33 ++++--- bbot/scanner/preset/environ.py | 94 +++++++------------ bbot/scanner/preset/preset.py | 43 +++++---- bbot/scanner/scanner.py | 7 +- .../test_manager_scope_accuracy.py | 14 ++- bbot/test/test_step_1/test_presets.py | 2 + .../test_module_asset_inventory.py | 6 +- .../module_tests/test_module_discord.py | 2 +- .../module_tests/test_module_http.py | 6 +- .../module_tests/test_module_httpx.py | 2 +- .../module_tests/test_module_json.py | 2 +- .../module_tests/test_module_nmap.py | 5 +- .../module_tests/test_module_slack.py | 2 +- .../module_tests/test_module_splunk.py | 2 +- .../module_tests/test_module_subdomains.py | 2 +- .../module_tests/test_module_teams.py | 2 +- .../module_tests/test_module_websocket.py | 2 +- 28 files changed, 146 insertions(+), 158 deletions(-) diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py index 1c8debc1e..03e318683 100644 --- a/bbot/agent/agent.py +++ b/bbot/agent/agent.py @@ -123,7 +123,7 @@ async def start_scan(self, scan_id, name=None, targets=[], modules=[], output_mo f"Starting scan with targets={targets}, modules={modules}, output_modules={output_modules}" ) output_module_config = OmegaConf.create( - {"output_modules": {"websocket": {"url": f"{self.url}/scan/{scan_id}/", "token": self.token}}} + {"modules": {"websocket": {"url": f"{self.url}/scan/{scan_id}/", "token": self.token}}} ) config = OmegaConf.create(config) config = OmegaConf.merge(self.config, config, output_module_config) diff --git a/bbot/cli.py b/bbot/cli.py index 449263bc3..0b0490f7c 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -27,6 +27,8 @@ async def _main(): preset = Preset() # parse command line arguments and merge into preset preset.parse_args() + # ensure arguments (-c config options etc.) are valid + preset.args.validate() # print help if no arguments if len(sys.argv) == 1: diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 79c3521aa..ec0cfb914 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -24,7 +24,7 @@ def ensure_config_files(self): mkdir(self.config_dir) secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] - exclude_keys = ["modules", "output_modules", "internal_modules"] + exclude_keys = ["modules"] comment_notice = ( "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" diff --git a/bbot/core/core.py b/bbot/core/core.py index bb509c0ad..17a2e92e8 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -38,6 +38,14 @@ def cache_dir(self): def tools_dir(self): return self.home / "tools" + @property + def temp_dir(self): + return self.home / "temp" + + @property + def lib_dir(self): + return self.home / "lib" + @property def scans_dir(self): return self.home / "scans" diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index c5fe716af..62a627033 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -43,11 +43,8 @@ def __init__(self, parent_helper): self.parent_helper.mkdir(self.command_status) self.setup_status = self.read_setup_status() - self.no_deps = self.parent_helper.config.get("no_deps", False) + self.deps_behavior = self.parent_helper.config.get("deps_behavior", "abort_on_failure").lower() self.ansible_debug = True - self.force_deps = self.parent_helper.config.get("force_deps", False) - self.retry_deps = self.parent_helper.config.get("retry_deps", False) - self.ignore_failed_deps = self.parent_helper.config.get("ignore_failed_deps", False) self.venv = "" if sys.prefix != sys.base_prefix: self.venv = sys.prefix @@ -62,7 +59,7 @@ async def install(self, *modules): notified = False for m in modules: # assume success if we're ignoring dependencies - if self.no_deps: + if self.deps_behavior == "no_deps": succeeded.append(m) continue # abort if module name is unknown @@ -87,7 +84,11 @@ async def install(self, *modules): succeeded.append(m) continue else: - if success is None or (success is False and self.retry_deps) or self.force_deps: + if ( + success is None + or (success is False and self.deps_behavior == "retry_failed") + or self.deps_behavior == "force_install" + ): if not notified: log.hugeinfo(f"Installing module dependencies. Please be patient, this may take a while.") notified = True @@ -97,14 +98,14 @@ async def install(self, *modules): self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.') success = await self.install_module(m) self.setup_status[module_hash] = success - if success or self.ignore_failed_deps: + if success or self.deps_behavior == "ignore_failed": log.debug(f'Setup succeeded for module "{m}"') succeeded.append(m) else: log.warning(f'Setup failed for module "{m}"') failed.append(m) else: - if success or self.ignore_failed_deps: + if success or self.deps_behavior == "ignore_failed": log.debug( f'Skipping dependency install for module "{m}" because it\'s already done (--force-deps to re-run)' ) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index d416ac768..045cdf313 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -99,7 +99,7 @@ def make_target(self, *events): @property def config(self): - return self.preset.core.config + return self.preset.config @property def scan(self): diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 87df73e6d..ad8d6a8ef 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -476,7 +476,7 @@ async def curl(self, *args, **kwargs): # only add custom headers if the URL is in-scope if self.parent_helper.scan.in_scope(url): - for hk, hv in self.parent_helper.scan.config.get("http_headers", {}).items(): + for hk, hv in self.config.get("http_headers", {}).items(): headers[hk] = hv # add the timeout @@ -560,9 +560,9 @@ def is_spider_danger(self, source_event, url): False """ url_depth = self.parent_helper.url_depth(url) - web_spider_depth = self.parent_helper.scan.config.get("web_spider_depth", 1) + web_spider_depth = self.config.get("web_spider_depth", 1) spider_distance = getattr(source_event, "web_spider_distance", 0) + 1 - web_spider_distance = self.parent_helper.scan.config.get("web_spider_distance", 0) + web_spider_distance = self.config.get("web_spider_distance", 0) if (url_depth > web_spider_depth) or (spider_distance > web_spider_distance): return True return False diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 42b1ce862..97bfa8441 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -6,6 +6,7 @@ import logging import importlib import traceback +from copy import copy from pathlib import Path from omegaconf import OmegaConf from contextlib import suppress @@ -42,7 +43,6 @@ def __init__(self): self.core = CORE self.__preloaded = {} - self._preloaded_orig = None self._modules = {} self._configs = {} @@ -55,6 +55,11 @@ def __init__(self): # save preload cache before exiting atexit.register(self.save_preload_cache) + def copy(self): + module_loader_copy = copy(self) + module_loader_copy.__preloaded = dict(self.__preloaded) + return module_loader_copy + @property def preload_cache_file(self): return self.core.cache_dir / "module_preload_cache" @@ -209,9 +214,7 @@ def configs(self, type=None): return OmegaConf.create(configs) def find_and_replace(self, **kwargs): - if self._preloaded_orig is None: - self._preloaded_orig = dict(self._preloaded) - self.__preloaded = search_format_dict(self._preloaded_orig, **kwargs) + self.__preloaded = search_format_dict(self.__preloaded, **kwargs) def check_type(self, module, type): return self._preloaded[module]["type"] == type @@ -541,14 +544,10 @@ def modules_options(self, modules=None, mod_type=None): modules_options = {} for module_name, preloaded in self.filter_modules(modules, mod_type): modules_options[module_name] = [] - module_type = preloaded["type"] module_options = preloaded["config"] module_options_desc = preloaded["options_desc"] for k, v in sorted(module_options.items(), key=lambda x: x[0]): - module_key = "modules" - if module_type in ("internal", "output"): - module_key = f"{module_type}_modules" - option_name = f"{module_key}.{module_name}.{k}" + option_name = f"modules.{module_name}.{k}" option_type = type(v).__name__ option_description = module_options_desc[k] modules_options[module_name].append((option_name, option_type, option_description, str(v))) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index df84bcff1..d29d6ecbb 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -49,6 +49,14 @@ excavate: True # Summarize activity at the end of a scan aggregate: True +# How to handle installation of module dependencies +# Choices are: +# - abort_on_failure (default) - if a module dependency fails to install, abort the scan +# - retry_failed - try again to install failed dependencies +# - ignore_failed - run the scan regardless of what happens with dependency installation +# - disable - completely disable BBOT's dependency system (you are responsible for install tools, pip packages, etc.) +deps_behavior: abort_on_failure + # HTTP timeout (for Python requests; API calls, etc.) http_timeout: 10 # HTTP timeout (for httpx) diff --git a/bbot/modules/internal/base.py b/bbot/modules/internal/base.py index 9e7967b42..8ef1b7fd9 100644 --- a/bbot/modules/internal/base.py +++ b/bbot/modules/internal/base.py @@ -9,13 +9,6 @@ class BaseInternalModule(BaseModule): # Priority, 1-5, lower numbers == higher priority _priority = 3 - @property - def config(self): - config = self.scan.config.get("internal_modules", {}).get(self.name, {}) - if config is None: - config = {} - return config - @property def log(self): if self._log is None: diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 1d4ec9a36..f8e2f74c6 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -74,13 +74,6 @@ def file(self): self._file = open(self.output_file, mode="a") return self._file - @property - def config(self): - config = self.scan.config.get("output_modules", {}).get(self.name, {}) - if config is None: - config = {} - return config - @property def log(self): if self._log is None: diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index ff0cf5ed2..44ca90262 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -125,11 +125,22 @@ def preset_from_args(self): # verbosity levels if self.parsed.silent: - args_preset.silent = self.parsed.silent + args_preset.silent = True if self.parsed.verbose: - args_preset.verbose = self.parsed.verbose + args_preset.verbose = True if self.parsed.debug: - args_preset.debug = self.parsed.debug + args_preset.debug = True + + # dependencies + if self.parsed.retry_deps: + args_preset.core.custom_config["deps_behavior"] = "retry_failed" + elif self.parsed.force_deps: + args_preset.core.custom_config["deps_behavior"] = "force_install" + elif self.parsed.no_deps: + args_preset.core.custom_config["deps_behavior"] = "disable" + elif self.parsed.ignore_failed_deps: + args_preset.core.custom_config["deps_behavior"] = "ignore_failed" + return args_preset def create_parser(self, *args, **kwargs): @@ -289,30 +300,26 @@ def validate(self): if self.exclude_from_validation.match(c): continue if all_options is None: - from ...modules import module_loader - modules_options = set() - for module_options in module_loader.modules_options().values(): + for module_options in self.preset.module_loader.modules_options().values(): modules_options.update(set(o[0] for o in module_options)) - global_options = set(self.preset.core.default_config.keys()) - {"modules", "output_modules"} + global_options = set(self.preset.core.default_config.keys()) - {"modules"} all_options = global_options.union(modules_options) # otherwise, ensure it exists as a module option match_and_exit(c, all_options, msg="module option") - # PRESET TODO: if custom module dir, pull in new module choices - # validate modules - for m in self.parser.modules: + for m in self.parsed.modules: if m not in self._module_choices: match_and_exit(m, self._module_choices, msg="module") - for m in self.parser.exclude_modules: + for m in self.parsed.exclude_modules: if m not in self._module_choices: match_and_exit(m, self._module_choices, msg="module") - for m in self.parser.output_modules: + for m in self.parsed.output_modules: if m not in self._output_module_choices: match_and_exit(m, self._output_module_choices, msg="output module") # validate flags - for f in set(self.parser.flags + self.parser.require_flags + self.parser.exclude_flags): + for f in set(self.parsed.flags + self.parsed.require_flags + self.parsed.exclude_flags): if f not in self._flag_choices: match_and_exit(f, self._flag_choices, msg="flag") diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 89d21987d..21f951bd3 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -26,8 +26,8 @@ def increase_limit(new_limit): class BBOTEnviron: - def __init__(self, core): - self.core = core + def __init__(self, preset): + self.preset = preset def flatten_config(self, config, base="bbot"): """ @@ -42,98 +42,72 @@ def flatten_config(self, config, base="bbot"): elif type(v) != omegaconf.listconfig.ListConfig: yield (new_base.upper(), str(v)) - def add_to_path(self, v, k="PATH"): + def add_to_path(self, v, k="PATH", environ=None): + """ + Add an entry to a colon-separated PATH variable. + If it's already contained in the value, shift it to be in first position. + """ + if environ is None: + environ = os.environ var_list = os.environ.get(k, "").split(":") deduped_var_list = [] for _ in var_list: - if not _ in deduped_var_list: + if _ != v and _ not in deduped_var_list: deduped_var_list.append(_) - if not v in deduped_var_list: - deduped_var_list = [v] + deduped_var_list + deduped_var_list = [v] + deduped_var_list new_var_str = ":".join(deduped_var_list) - os.environ[k] = new_var_str + environ[k] = new_var_str def prepare(self): """ Sync config to OS environment variables """ + environ = dict(os.environ) + # if we're running in a virtual environment, make sure to include its /bin in PATH if sys.prefix != sys.base_prefix: bin_dir = str(Path(sys.prefix) / "bin") - self.add_to_path(bin_dir) + self.add_to_path(bin_dir, environ=environ) # add ~/.local/bin to PATH local_bin_dir = str(Path.home() / ".local" / "bin") - self.add_to_path(local_bin_dir) + self.add_to_path(local_bin_dir, environ=environ) # ensure bbot_tools - bbot_tools = self.core.home / "tools" - os.environ["BBOT_TOOLS"] = str(bbot_tools) - if not str(bbot_tools) in os.environ.get("PATH", "").split(":"): - os.environ["PATH"] = f'{bbot_tools}:{os.environ.get("PATH", "").strip(":")}' + environ["BBOT_TOOLS"] = str(self.preset.core.tools_dir) + self.add_to_path(str(self.preset.core.tools_dir), environ=environ) # ensure bbot_cache - bbot_cache = self.core.home / "cache" - os.environ["BBOT_CACHE"] = str(bbot_cache) + environ["BBOT_CACHE"] = str(self.preset.core.cache_dir) # ensure bbot_temp - bbot_temp = self.core.home / "temp" - os.environ["BBOT_TEMP"] = str(bbot_temp) + environ["BBOT_TEMP"] = str(self.preset.core.temp_dir) # ensure bbot_lib - bbot_lib = self.core.home / "lib" - os.environ["BBOT_LIB"] = str(bbot_lib) + environ["BBOT_LIB"] = str(self.preset.core.lib_dir) # export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:~/.bbot/lib/ - self.add_to_path(str(bbot_lib), k="LD_LIBRARY_PATH") + self.add_to_path(str(self.preset.core.lib_dir), k="LD_LIBRARY_PATH", environ=environ) # platform variables - os.environ["BBOT_OS_PLATFORM"] = os_platform() - os.environ["BBOT_OS"] = os_platform_friendly() - os.environ["BBOT_CPU_ARCH"] = cpu_architecture() - - # exchange certain options between CLI args and config - # PRESET TODO: do we even need this? - # if self.core.cli_execution and self.core.args.cli_config: - # # deps - # self.core.custom_config["retry_deps"] = self.core.args.cli_options.retry_deps - # self.core.custom_config["force_deps"] = self.core.args.cli_options.force_deps - # self.core.custom_config["no_deps"] = self.core.args.cli_options.no_deps - # self.core.custom_config["ignore_failed_deps"] = self.core.args.cli_options.ignore_failed_deps - # # debug - # self.core.custom_config["debug"] = self.core.args.cli_options.debug - # self.core.custom_config["silent"] = self.core.args.cli_options.silent - - import logging - - log = logging.getLogger() - if self.core.custom_config.get("debug", False): - self.core.custom_config["silent"] = False - log = logging.getLogger("bbot") - log.setLevel(logging.DEBUG) - logging.getLogger("asyncio").setLevel(logging.DEBUG) - elif self.core.custom_config.get("silent", False): - log = logging.getLogger("bbot") - log.setLevel(logging.CRITICAL) + environ["BBOT_OS_PLATFORM"] = os_platform() + environ["BBOT_OS"] = os_platform_friendly() + environ["BBOT_CPU_ARCH"] = cpu_architecture() # copy config to environment - bbot_environ = self.flatten_config(self.core.config) - os.environ.update(bbot_environ) + bbot_environ = self.flatten_config(self.preset.config) + environ.update(bbot_environ) # handle HTTP proxy - http_proxy = self.core.config.get("http_proxy", "") + http_proxy = self.preset.config.get("http_proxy", "") if http_proxy: - os.environ["HTTP_PROXY"] = http_proxy - os.environ["HTTPS_PROXY"] = http_proxy + environ["HTTP_PROXY"] = http_proxy + environ["HTTPS_PROXY"] = http_proxy else: - os.environ.pop("HTTP_PROXY", None) - os.environ.pop("HTTPS_PROXY", None) - - # replace environment variables in preloaded modules - # PRESET TODO: move this - # self.core.module_loader.find_and_replace(**os.environ) + environ.pop("HTTP_PROXY", None) + environ.pop("HTTPS_PROXY", None) # ssl verification import urllib3 urllib3.disable_warnings() - ssl_verify = self.core.config.get("ssl_verify", False) + ssl_verify = self.preset.config.get("ssl_verify", False) if not ssl_verify: import requests import functools @@ -146,3 +120,5 @@ def prepare(self): ) requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) requests.request = functools.partial(requests.request, verify=False) + + return environ diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 77b0ce343..9e20c4f8b 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,6 +1,8 @@ +import os import yaml import logging import omegaconf +from copy import copy from pathlib import Path from contextlib import suppress @@ -79,9 +81,6 @@ def __init__( self.output_modules = output_modules if output_modules is not None else [] self.internal_modules = internal_modules if internal_modules is not None else [] - # PRESET TODO: preparation of environment - # self.core.environ.prepare() - self.strict_scope = strict_scope # target / whitelist / blacklist @@ -141,6 +140,26 @@ def merge(self, other): if other.debug: self.debug = other.debug + def bake(self): + """ + return a "baked" copy of the preset, ready for use by a BBOT scan + """ + # create a copy of self + baked_preset = copy(self) + # copy core + baked_preset.core = self.core.copy() + # copy module loader + baked_preset._module_loader = self.module_loader.copy() + # prepare os environment + os_environ = baked_preset.environ.prepare() + # find and replace preloaded modules with os environ + baked_preset.module_loader.find_and_replace(**os_environ) + # update os environ + os.environ.clear() + os.environ.update(os_environ) + + return baked_preset + def parse_args(self): from .args import BBOTArgs @@ -148,17 +167,6 @@ def parse_args(self): self._args = BBOTArgs(self) self.merge(self.args.preset_from_args()) - # bring in presets - # self.merge(self.args.presets) - - # bring in config - # self.core.merge_custom(self.args.config) - - # bring in misc cli arguments - - # validate config / modules / flags - # self.args.validate() - @property def module_dirs(self): return self.module_loader.module_dirs @@ -397,9 +405,6 @@ def helpers(self): @property def module_loader(self): - # module loader depends on environment to be set up - # or is it the other way around - # PRESET TODO self.environ if self._module_loader is None: from bbot.core.modules import module_loader @@ -414,9 +419,7 @@ def refresh_module_loader(self): # update default config with module defaults module_config = omegaconf.OmegaConf.create( { - "modules": self.module_loader.configs(type="scan"), - "output_modules": self.module_loader.configs(type="output"), - "internal_modules": self.module_loader.configs(type="internal"), + "modules": self.module_loader.configs(), } ) self.core.merge_default(module_config) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 423af6545..afe6d210c 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -133,10 +133,9 @@ def __init__( self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" preset = preset_kwargs.pop("preset", None) - if preset is not None: - self.preset = preset - else: - self.preset = Preset(*args, **preset_kwargs) + if preset is None: + preset = Preset(*args, **preset_kwargs) + self.preset = preset.bake() # scan name if scan_name is None: diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index de9ffd72c..f65da3194 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -311,7 +311,7 @@ def custom_setup(scan): "scope_dns_search_distance": 2, "scope_report_distance": 1, "speculate": True, - "internal_modules": {"speculate": {"ports": "8888"}}, + "modules": {"speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, ) @@ -374,8 +374,7 @@ def custom_setup(scan): "scope_dns_search_distance": 2, "scope_report_distance": 1, "speculate": True, - "modules": {"httpx": {"in_scope_only": False}}, - "internal_modules": {"speculate": {"ports": "8888"}}, + "modules": {"httpx": {"in_scope_only": False}, "speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, ) @@ -454,8 +453,7 @@ def custom_setup(scan): "scope_dns_search_distance": 2, "scope_report_distance": 1, "speculate": True, - "modules": {"httpx": {"in_scope_only": False}}, - "internal_modules": {"speculate": {"ports": "8888"}}, + "modules": {"httpx": {"in_scope_only": False}, "speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, ) @@ -548,7 +546,7 @@ def custom_setup(scan): "scope_dns_search_distance": 2, "scope_report_distance": 0, "speculate": True, - "internal_modules": {"speculate": {"ports": "8888"}}, + "modules": {"speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, ) @@ -666,7 +664,7 @@ def custom_setup(scan): events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.0/31", modules=["speculate", "sslcert"], - _config={"dns_resolution": False, "scope_report_distance": 0, "internal_modules": {"speculate": {"ports": "9999"}}}, + _config={"dns_resolution": False, "scope_report_distance": 0, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) @@ -723,7 +721,7 @@ def custom_setup(scan): "127.0.0.0/31", modules=["speculate", "sslcert"], whitelist=["127.0.1.0"], - _config={"dns_resolution": False, "scope_report_distance": 0, "internal_modules": {"speculate": {"ports": "9999"}}}, + _config={"dns_resolution": False, "scope_report_distance": 0, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 5ebe9a7ec..e1d4e3683 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -411,3 +411,5 @@ class TestModule4(BaseModule): # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation +# what if you specify -c modules.custommodule.option +# the validation needs to not happen until after all presets have been loaded diff --git a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py index 5e8c9b3a1..af46ad5ba 100644 --- a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +++ b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py @@ -4,7 +4,7 @@ class TestAsset_Inventory(ModuleTestBase): targets = ["127.0.0.1", "bbottest.notreal"] scan_name = "asset_inventory_test" - config_overrides = {"dns_resolution": True, "internal_modules": {"nmap": {"ports": "9999"}}} + config_overrides = {"dns_resolution": True, "modules": {"nmap": {"ports": "9999"}}} modules_overrides = ["asset_inventory", "nmap", "sslcert"] async def setup_before_prep(self, module_test): @@ -32,7 +32,7 @@ def check(self, module_test, events): class TestAsset_InventoryEmitPrevious(TestAsset_Inventory): - config_overrides = {"dns_resolution": True, "output_modules": {"asset_inventory": {"use_previous": True}}} + config_overrides = {"dns_resolution": True, "modules": {"asset_inventory": {"use_previous": True}}} modules_overrides = ["asset_inventory"] def check(self, module_test, events): @@ -54,7 +54,7 @@ def check(self, module_test, events): class TestAsset_InventoryRecheck(TestAsset_Inventory): config_overrides = { "dns_resolution": True, - "output_modules": {"asset_inventory": {"use_previous": True, "recheck": True}}, + "modules": {"asset_inventory": {"use_previous": True, "recheck": True}}, } modules_overrides = ["asset_inventory"] diff --git a/bbot/test/test_step_2/module_tests/test_module_discord.py b/bbot/test/test_step_2/module_tests/test_module_discord.py index 2a4143852..c0a210720 100644 --- a/bbot/test/test_step_2/module_tests/test_module_discord.py +++ b/bbot/test/test_step_2/module_tests/test_module_discord.py @@ -8,7 +8,7 @@ class TestDiscord(ModuleTestBase): modules_overrides = ["discord", "excavate", "badsecrets", "httpx"] webhook_url = "https://discord.com/api/webhooks/1234/deadbeef-P-uF-asdf" - config_overrides = {"output_modules": {"discord": {"webhook_url": webhook_url}}} + config_overrides = {"modules": {"discord": {"webhook_url": webhook_url}}} def custom_setup(self, module_test): respond_args = { diff --git a/bbot/test/test_step_2/module_tests/test_module_http.py b/bbot/test/test_step_2/module_tests/test_module_http.py index d0afcefb2..43b7189ad 100644 --- a/bbot/test/test_step_2/module_tests/test_module_http.py +++ b/bbot/test/test_step_2/module_tests/test_module_http.py @@ -7,7 +7,7 @@ class TestHTTP(ModuleTestBase): downstream_url = "https://blacklanternsecurity.fakedomain:1234/events" config_overrides = { - "output_modules": { + "modules": { "http": { "url": downstream_url, "method": "PUT", @@ -56,8 +56,8 @@ def check(self, module_test, events): class TestHTTPSIEMFriendly(TestHTTP): modules_overrides = ["http"] - config_overrides = {"output_modules": {"http": dict(TestHTTP.config_overrides["output_modules"]["http"])}} - config_overrides["output_modules"]["http"]["siem_friendly"] = True + config_overrides = {"modules": {"http": dict(TestHTTP.config_overrides["modules"]["http"])}} + config_overrides["modules"]["http"]["siem_friendly"] = True def verify_data(self, j): return j["data"] == {"DNS_NAME": "blacklanternsecurity.com"} and j["type"] == "DNS_NAME" diff --git a/bbot/test/test_step_2/module_tests/test_module_httpx.py b/bbot/test/test_step_2/module_tests/test_module_httpx.py index ebd9bbdb1..f67525aeb 100644 --- a/bbot/test/test_step_2/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_2/module_tests/test_module_httpx.py @@ -56,7 +56,7 @@ def check(self, module_test, events): class TestHTTPX_404(ModuleTestBase): targets = ["https://127.0.0.1:9999"] modules_overrides = ["httpx", "speculate", "excavate"] - config_overrides = {"internal_modules": {"speculate": {"ports": "8888,9999"}}} + config_overrides = {"modules": {"speculate": {"ports": "8888,9999"}}} async def setup_after_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data( diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 1e67db085..03b1f6a24 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -16,7 +16,7 @@ def check(self, module_test, events): class TestJSONSIEMFriendly(ModuleTestBase): modules_overrides = ["json"] - config_overrides = {"output_modules": {"json": {"siem_friendly": True}}} + config_overrides = {"modules": {"json": {"siem_friendly": True}}} def check(self, module_test, events): txt_file = module_test.scan.home / "output.ndjson" diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py index 092f84a47..c115e0abe 100644 --- a/bbot/test/test_step_2/module_tests/test_module_nmap.py +++ b/bbot/test/test_step_2/module_tests/test_module_nmap.py @@ -29,8 +29,7 @@ def check(self, module_test, events): class TestNmapAssetInventory(ModuleTestBase): targets = ["127.0.0.1/31"] config_overrides = { - "modules": {"nmap": {"ports": "8888,8889"}}, - "output_modules": {"asset_inventory": {"use_previous": True}}, + "modules": {"nmap": {"ports": "8888,8889"}, "asset_inventory": {"use_previous": True}}, } modules_overrides = ["nmap", "asset_inventory"] module_name = "nmap" @@ -40,7 +39,7 @@ async def setup_after_prep(self, module_test): from bbot.scanner import Scanner first_scan_config = module_test.scan.config.copy() - first_scan_config["output_modules"]["asset_inventory"]["use_previous"] = False + first_scan_config["modules"]["asset_inventory"]["use_previous"] = False first_scan = Scanner("127.0.0.1", name=self.scan_name, modules=["asset_inventory"], config=first_scan_config) await first_scan.async_start_without_generator() diff --git a/bbot/test/test_step_2/module_tests/test_module_slack.py b/bbot/test/test_step_2/module_tests/test_module_slack.py index b486d7df2..2efea1478 100644 --- a/bbot/test/test_step_2/module_tests/test_module_slack.py +++ b/bbot/test/test_step_2/module_tests/test_module_slack.py @@ -6,4 +6,4 @@ class TestSlack(DiscordBase): modules_overrides = ["slack", "excavate", "badsecrets", "httpx"] webhook_url = "https://hooks.slack.com/services/deadbeef/deadbeef/deadbeef" - config_overrides = {"output_modules": {"slack": {"webhook_url": webhook_url}}} + config_overrides = {"modules": {"slack": {"webhook_url": webhook_url}}} diff --git a/bbot/test/test_step_2/module_tests/test_module_splunk.py b/bbot/test/test_step_2/module_tests/test_module_splunk.py index 67d67a4ef..d55ed17c2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_splunk.py +++ b/bbot/test/test_step_2/module_tests/test_module_splunk.py @@ -7,7 +7,7 @@ class TestSplunk(ModuleTestBase): downstream_url = "https://splunk.blacklanternsecurity.fakedomain:1234/services/collector" config_overrides = { - "output_modules": { + "modules": { "splunk": { "url": downstream_url, "hectoken": "HECTOKEN", diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomains.py b/bbot/test/test_step_2/module_tests/test_module_subdomains.py index 9aa9f7b5e..65b9a8a03 100644 --- a/bbot/test/test_step_2/module_tests/test_module_subdomains.py +++ b/bbot/test/test_step_2/module_tests/test_module_subdomains.py @@ -17,7 +17,7 @@ def check(self, module_test, events): class TestSubdomainsUnresolved(TestSubdomains): - config_overrides = {"output_modules": {"subdomains": {"include_unresolved": True}}} + config_overrides = {"modules": {"subdomains": {"include_unresolved": True}}} def check(self, module_test, events): sub_file = module_test.scan.home / "subdomains.txt" diff --git a/bbot/test/test_step_2/module_tests/test_module_teams.py b/bbot/test/test_step_2/module_tests/test_module_teams.py index f544f5cb9..89344b680 100644 --- a/bbot/test/test_step_2/module_tests/test_module_teams.py +++ b/bbot/test/test_step_2/module_tests/test_module_teams.py @@ -8,7 +8,7 @@ class TestTeams(DiscordBase): modules_overrides = ["teams", "excavate", "badsecrets", "httpx"] webhook_url = "https://evilcorp.webhook.office.com/webhookb2/deadbeef@deadbeef/IncomingWebhook/deadbeef/deadbeef" - config_overrides = {"output_modules": {"teams": {"webhook_url": webhook_url}}} + config_overrides = {"modules": {"teams": {"webhook_url": webhook_url}}} async def setup_after_prep(self, module_test): self.custom_setup(module_test) diff --git a/bbot/test/test_step_2/module_tests/test_module_websocket.py b/bbot/test/test_step_2/module_tests/test_module_websocket.py index d1620702c..fcf5c2eee 100644 --- a/bbot/test/test_step_2/module_tests/test_module_websocket.py +++ b/bbot/test/test_step_2/module_tests/test_module_websocket.py @@ -23,7 +23,7 @@ async def server_coroutine(): class TestWebsocket(ModuleTestBase): - config_overrides = {"output_modules": {"websocket": {"url": "ws://127.0.0.1:8765/testing"}}} + config_overrides = {"modules": {"websocket": {"url": "ws://127.0.0.1:8765/testing"}}} async def setup_before_prep(self, module_test): self.server_task = asyncio.create_task(server_coroutine()) From 4377dda7912e8a84109c09cc332cf0c82019dfc7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 17 Mar 2024 19:38:10 -0400 Subject: [PATCH 040/171] fix scope bug --- bbot/cli.py | 8 ++++++++ bbot/core/helpers/cloud.py | 4 ++-- bbot/core/helpers/dns.py | 3 +-- bbot/core/helpers/helper.py | 2 +- bbot/core/helpers/web.py | 8 ++++---- bbot/modules/baddns.py | 4 +--- bbot/modules/baddns_zone.py | 5 ----- bbot/modules/base.py | 14 +++++++++++--- bbot/scanner/manager.py | 3 ++- bbot/scanner/preset/args.py | 16 +++++++++------- bbot/scanner/preset/preset.py | 4 ++-- bbot/test/test_step_1/test_presets.py | 11 +++++++++-- 12 files changed, 50 insertions(+), 32 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 0b0490f7c..f4b64262b 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -74,6 +74,14 @@ async def _main(): scan = Scanner(preset=preset) + if not preset.args.parsed.dry_run: + log.trace(f"Command: {' '.join(sys.argv)}") + + if sys.stdin.isatty(): + if not preset.args.parsed.agent_mode and not preset.args.parsed.yes: + log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") + input() + await scan.async_start_without_generator() diff --git a/bbot/core/helpers/cloud.py b/bbot/core/helpers/cloud.py index 677efa093..7f1e19b69 100644 --- a/bbot/core/helpers/cloud.py +++ b/bbot/core/helpers/cloud.py @@ -29,7 +29,7 @@ def excavate(self, event, s): if not match in found: found.add(match) if event_type == "STORAGE_BUCKET": - self.emit_bucket(match, **kwargs) + yield self.emit_bucket(match, **kwargs) else: yield kwargs @@ -57,7 +57,7 @@ def speculate(self, event): else: yield kwargs - async def emit_bucket(self, match, **kwargs): + def emit_bucket(self, match, **kwargs): bucket_name, bucket_domain = match kwargs["data"] = {"name": bucket_name, "url": f"https://{bucket_name}.{bucket_domain}"} return kwargs diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 463a1bb5a..d7f680e13 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -141,8 +141,7 @@ def __init__(self, parent_helper): # copy the system's current resolvers to a text file for tool use self.system_resolvers = dns.resolver.Resolver().nameservers - if len(self.system_resolvers) == 1: - log.warning("BBOT performs better with multiple DNS servers. Your system currently only has one.") + # TODO: DNS server speed test (start in background task) self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False) self.filter_bad_ptrs = self.parent_helper.config.get("dns_filter_ptrs", True) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 045cdf313..8cc923fde 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -95,7 +95,7 @@ def clean_old_scans(self): self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter) def make_target(self, *events): - return Target(self.scan, *events) + return Target(*events) @property def config(self): diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index ad8d6a8ef..310aaca82 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -475,8 +475,8 @@ async def curl(self, *args, **kwargs): headers["User-Agent"] = user_agent # only add custom headers if the URL is in-scope - if self.parent_helper.scan.in_scope(url): - for hk, hv in self.config.get("http_headers", {}).items(): + if self.parent_helper.preset.in_scope(url): + for hk, hv in self.parent_helper.config.get("http_headers", {}).items(): headers[hk] = hv # add the timeout @@ -560,9 +560,9 @@ def is_spider_danger(self, source_event, url): False """ url_depth = self.parent_helper.url_depth(url) - web_spider_depth = self.config.get("web_spider_depth", 1) + web_spider_depth = self.parent_helper.config.get("web_spider_depth", 1) spider_distance = getattr(source_event, "web_spider_distance", 0) + 1 - web_spider_distance = self.config.get("web_spider_distance", 0) + web_spider_distance = self.parent_helper.config.get("web_spider_distance", 0) if (url_depth > web_spider_depth) or (spider_distance > web_spider_distance): return True return False diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 9abfebc84..992ae5c0d 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -4,9 +4,6 @@ import asyncio import logging -from bbot.core.logger.logger import include_logger - -include_logger(logging.getLogger("baddns")) class baddns(BaseModule): @@ -30,6 +27,7 @@ def select_modules(self): return selected_modules async def setup(self): + self.preset.core.logger.include_logger(logging.getLogger("baddns")) self.custom_nameservers = self.config.get("custom_nameservers", []) or None if self.custom_nameservers: self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index a42fe2e21..ac0fc3c57 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -1,11 +1,6 @@ from baddns.base import get_all_modules from .baddns import baddns as baddns_module -import logging -from bbot.core.logger.logger import include_logger - -include_logger(logging.getLogger("baddns_zone")) - class baddns_zone(baddns_module): watched_events = ["DNS_NAME"] diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 2a1c327d2..093fa9680 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -561,7 +561,7 @@ async def _setup(self): msg = status_codes[status] self.debug(f"Finished setting up module {self.name}") except Exception as e: - self.set_error_state() + self.set_error_state(f"Unexpected error during module setup: {e}", critical=True) msg = f"{e}" self.trace() return self.name, status, str(msg) @@ -843,7 +843,7 @@ async def queue_outgoing_event(self, event, **kwargs): except AttributeError: self.debug(f"Not in an acceptable state to queue outgoing event") - def set_error_state(self, message=None, clear_outgoing_queue=False): + def set_error_state(self, message=None, clear_outgoing_queue=False, critical=False): """ Puts the module into an errored state where it cannot accept new events. Optionally logs a warning message. @@ -868,7 +868,11 @@ def set_error_state(self, message=None, clear_outgoing_queue=False): log_msg = "Setting error state" if message is not None: log_msg += f": {message}" - self.warning(log_msg) + if critical: + log_fn = self.error + else: + log_fn = self.warning + log_fn(log_msg) self.errored = True # clear incoming queue if self.incoming_event_queue is not False: @@ -1071,6 +1075,10 @@ async def request_with_fail_count(self, *args, **kwargs): self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests") return r + @property + def preset(self): + return self.scan.preset + @property def config(self): """Property that provides easy access to the module's configuration in the scan's config. diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 90387eae6..81b0dbca9 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -38,6 +38,7 @@ def __init__(self, scan): """ self.scan = scan + self.preset = scan.preset self.incoming_event_queue = ShuffleQueue() # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules) @@ -317,7 +318,7 @@ async def _emit_event(self, event, **kwargs): if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: child_event.add_tag("affiliate") host_hash = hash(str(child_event.host)) - if in_dns_scope or self.scan.in_scope(child_event): + if in_dns_scope or self.preset.in_scope(child_event): dns_child_events.append(child_event) except ValidationError as e: log.warning( diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 44ca90262..f644d0bd9 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -1,5 +1,6 @@ import re import sys +import logging import argparse from pathlib import Path from omegaconf import OmegaConf @@ -7,6 +8,8 @@ from bbot.core.helpers.logger import log_to_stderr from bbot.core.helpers.misc import chain_lists, match_and_exit, is_file +log = logging.getLogger("bbot.presets.args") + class BBOTArgs: @@ -87,13 +90,12 @@ def parsed(self): return self._parsed def preset_from_args(self): - args_preset = self.preset.__class__() - - # scope - args_preset.target.add_target(self.parsed.targets) - args_preset.whitelist.add_target(self.parsed.whitelist) - args_preset.blacklist.add_target(self.parsed.blacklist) - args_preset.strict_scope = self.parsed.strict_scope + args_preset = self.preset.__class__( + *self.parsed.targets, + whitelist=self.parsed.whitelist, + blacklist=self.parsed.blacklist, + strict_scope=self.parsed.strict_scope, + ) # modules && flags (excluded then required then all others) args_preset.exclude_modules = self.parsed.exclude_modules diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9e20c4f8b..11df32556 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -127,7 +127,7 @@ def merge(self, other): self.add_module(module_name, module_type=module_type) # scope self.target.add_target(other.target) - self.whitelist = other.whitelist + self.whitelist.add_target(other.whitelist) self.blacklist.add_target(other.blacklist) self.strict_scope = self.strict_scope or other.strict_scope for t in (self.target, self.whitelist): @@ -446,7 +446,7 @@ def in_scope(self, e): Examples: Check if a URL is in scope: - >>> scan.in_scope("http://www.evilcorp.com") + >>> preset.in_scope("http://www.evilcorp.com") True """ try: diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index e1d4e3683..e7dd99452 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -71,6 +71,12 @@ def test_preset_yaml(): assert "evilcorp.ce" in preset1.whitelist assert "test.www.evilcorp.ce" in preset1.blacklist assert "sslcert" in preset1.scan_modules + assert preset1.whitelisted("evilcorp.ce") + assert preset1.whitelisted("www.evilcorp.ce") + assert not preset1.whitelisted("evilcorp.com") + assert preset1.blacklisted("test.www.evilcorp.ce") + assert preset1.blacklisted("asdf.test.www.evilcorp.ce") + assert not preset1.blacklisted("www.evilcorp.ce") # test yaml save/load yaml1 = preset1.to_yaml(sort_keys=True) @@ -127,13 +133,14 @@ def test_preset_scope(): # strict scope is enabled assert not "asdf.evilcorp.com" in preset1.target assert not "asdf.www.evilcorp.ce" in preset1.target - # whitelist is overridden, not merged - assert not "evilcorp.ce" in preset1.whitelist + assert "evilcorp.ce" in preset1.whitelist assert "evilcorp.de" in preset1.whitelist assert not "asdf.evilcorp.de" in preset1.whitelist + assert not "asdf.evilcorp.ce" in preset1.whitelist # blacklist should be merged, strict scope does not apply assert "asdf.test.www.evilcorp.ce" in preset1.blacklist assert "asdf.test.www.evilcorp.de" in preset1.blacklist + assert not "asdf.test.www.evilcorp.org" in preset1.blacklist # only the base domain of evilcorp.de should be in scope assert not preset1.in_scope("evilcorp.com") assert not preset1.in_scope("evilcorp.org") From e2f1707394cc791cf211c7fc24def8a4527f591a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 17 Mar 2024 19:41:27 -0400 Subject: [PATCH 041/171] create preload cache parent dir --- bbot/core/modules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 97bfa8441..e662665e9 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -15,7 +15,7 @@ from .flags import flag_descriptions from .helpers.logger import log_to_stderr -from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform +from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform, mkdir log = logging.getLogger("bbot.module_loader") @@ -177,6 +177,7 @@ def preload_cache(self): @preload_cache.setter def preload_cache(self, value): self._preload_cache = value + mkdir(self.self.preload_cache_file.parent) with open(self.preload_cache_file, "wb") as f: pickle.dump(self._preload_cache, f) From 2c0590f03ad1f158206681a4a3bb8a55f140425b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 17 Mar 2024 22:32:09 -0400 Subject: [PATCH 042/171] fix module test tests --- bbot/core/modules.py | 2 +- bbot/test/test_step_1/test__module__tests.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index e662665e9..0f6563f97 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -177,7 +177,7 @@ def preload_cache(self): @preload_cache.setter def preload_cache(self, value): self._preload_cache = value - mkdir(self.self.preload_cache_file.parent) + mkdir(self.preload_cache_file.parent) with open(self.preload_cache_file, "wb") as f: pickle.dump(self._preload_cache, f) diff --git a/bbot/test/test_step_1/test__module__tests.py b/bbot/test/test_step_1/test__module__tests.py index 8a225ef68..9d88b1bcc 100644 --- a/bbot/test/test_step_1/test__module__tests.py +++ b/bbot/test/test_step_1/test__module__tests.py @@ -2,7 +2,7 @@ import importlib from pathlib import Path -from bbot.core import CORE +from bbot.scanner import Preset from ..test_step_2.module_tests.base import ModuleTestBase log = logging.getLogger("bbot.test.modules") @@ -15,8 +15,11 @@ def test__module__tests(): + + preset = Preset() + # make sure each module has a .py file - for module_name in CORE.module_loader.preloaded(): + for module_name in preset.module_loader.preloaded(): module_name = module_name.lower() assert module_name in module_test_files, f'No test file found for module "{module_name}"' From e61963104374f0ffcf79761c643747302df1aeb4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 17 Mar 2024 23:42:11 -0400 Subject: [PATCH 043/171] tolerant preset loading --- bbot/core/errors.py | 4 +++ bbot/core/modules.py | 1 - bbot/scanner/preset/args.py | 43 ++++++++++++++++------------- bbot/scanner/preset/path.py | 51 +++++++++++++++++++++++++++++++++++ bbot/scanner/preset/preset.py | 10 ++++++- 5 files changed, 88 insertions(+), 21 deletions(-) create mode 100644 bbot/scanner/preset/path.py diff --git a/bbot/core/errors.py b/bbot/core/errors.py index d86899db2..ce4e3dcc9 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -48,3 +48,7 @@ class DNSWildcardBreak(DNSError): class CurlError(BBOTError): pass + + +class PresetNotFoundError(BBOTError): + pass diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 0f6563f97..556bfab79 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -116,7 +116,6 @@ def preload(self, module_dirs=None): for module_dir in module_dirs: if module_dir in self._module_dirs_preloaded: - log.debug(f'Custom module dir "{module_dir}" was already added') continue for module_file in list_files(module_dir, filter=self.file_filter): diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index f644d0bd9..7c69fd386 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -2,11 +2,11 @@ import sys import logging import argparse -from pathlib import Path from omegaconf import OmegaConf +from bbot.core.errors import PresetNotFoundError from bbot.core.helpers.logger import log_to_stderr -from bbot.core.helpers.misc import chain_lists, match_and_exit, is_file +from bbot.core.helpers.misc import chain_lists, match_and_exit log = logging.getLogger("bbot.presets.args") @@ -97,6 +97,14 @@ def preset_from_args(self): strict_scope=self.parsed.strict_scope, ) + # verbosity levels + if self.parsed.silent: + args_preset.silent = True + if self.parsed.verbose: + args_preset.verbose = True + if self.parsed.debug: + args_preset.debug = True + # modules && flags (excluded then required then all others) args_preset.exclude_modules = self.parsed.exclude_modules args_preset.exclude_flags = self.parsed.exclude_flags @@ -109,29 +117,27 @@ def preset_from_args(self): args_preset.flags = self.parsed.flags # additional custom presets / config options + preset_args = [] for preset_param in self.parsed.preset: - if Path(preset_param).is_file(): - try: - custom_preset = self.preset.from_yaml_file(preset_param) - except Exception as e: - log_to_stderr(f"Error parsing custom config at {preset_param}: {e}", level="ERROR") - sys.exit(2) + try: + # first try to load as a file + custom_preset = self.preset.from_yaml_file(preset_param) args_preset.merge(custom_preset) - else: + except PresetNotFoundError as e: + log.debug(e) try: + # if that fails, try to parse as key=value syntax cli_config = OmegaConf.from_cli([preset_param]) + preset_args.append(preset_param) + args_preset.core.merge_custom(cli_config) except Exception as e: log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") sys.exit(2) - args_preset.core.merge_custom(cli_config) + except Exception as e: + log_to_stderr(f"Error parsing custom config at {preset_param}: {e}", level="ERROR") + sys.exit(2) - # verbosity levels - if self.parsed.silent: - args_preset.silent = True - if self.parsed.verbose: - args_preset.verbose = True - if self.parsed.debug: - args_preset.debug = True + self.parsed.preset = preset_args # dependencies if self.parsed.retry_deps: @@ -291,9 +297,8 @@ def sanitize_args(self): def validate(self): # validate config options sentinel = object() - conf = [a for a in self.parsed.preset if not is_file(a)] all_options = None - for c in conf: + for c in self.parsed.preset: c = c.split("=")[0].strip() v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel) # if option isn't in the default config diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py new file mode 100644 index 000000000..a382f206e --- /dev/null +++ b/bbot/scanner/preset/path.py @@ -0,0 +1,51 @@ +import logging +from pathlib import Path + +from bbot.core.errors import PresetNotFoundError + +log = logging.getLogger("bbot.presets.path") + +DEFAULT_PRESET_PATH = Path(__file__).parent.parent.parent / "presets" + + +class PresetPath: + """ + Keeps track of where to look for preset .yaml files + """ + + def __init__(self): + self.paths = [DEFAULT_PRESET_PATH] + self.add_path(Path.cwd()) + + def find(self, filename): + filename = Path(filename) + self.add_path(filename.parent) + self.add_path(filename.parent / "presets") + if filename.is_file(): + log.hugesuccess(filename) + return filename + extension = filename.suffix.lower() + file_candidates = set() + for ext in (".yaml", ".yml"): + if extension != ext: + file_candidates.add(f"{filename.stem}{ext}") + file_candidates = sorted(file_candidates) + log.debug(f"Searching for preset in {self.paths}, file candidates: {file_candidates}") + for path in self.paths: + for candidate in file_candidates: + for file in path.rglob(candidate): + log.verbose(f'Found preset matching "{filename}" at {file}') + return file + raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') + + def add_path(self, path): + path = Path(path) + if path in self.paths: + return + if not path.is_dir(): + log.debug(f'Path "{path}" is not a directory') + return + self.paths.append(path) + + +PRESET_PATH = PresetPath() diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 11df32556..af73f2fac 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -6,6 +6,8 @@ from pathlib import Path from contextlib import suppress +from .path import PRESET_PATH + from bbot.core import CORE from bbot.core.event.base import make_event from bbot.core.errors import ValidationError @@ -492,7 +494,13 @@ def from_dict(cls, preset_dict): return new_preset @classmethod - def from_yaml_file(cls, yaml_preset): + def from_yaml_file(cls, filename): + """ + Create a preset from a YAML file. If the full path is not specified, BBOT will look in all the usual places for it. + + Specifying the file extension is optional. + """ + yaml_preset = PRESET_PATH.find(filename) return cls.from_dict(omegaconf.OmegaConf.load(yaml_preset)) @classmethod From f65be4ae7e985d42da693eb67c83bc93e9bc9cdb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 18 Mar 2024 07:31:48 -0400 Subject: [PATCH 044/171] prep scan before enter prompt --- bbot/cli.py | 19 ++++++++++++------- bbot/presets/subdomain-enum.yml | 5 +++++ 2 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 bbot/presets/subdomain-enum.yml diff --git a/bbot/cli.py b/bbot/cli.py index f4b64262b..d3eab1b14 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -23,12 +23,15 @@ async def _main(): from bbot.scanner import Scanner from bbot.scanner.preset import Preset + global scan_name + # start by creating a default scan preset preset = Preset() # parse command line arguments and merge into preset preset.parse_args() # ensure arguments (-c config options etc.) are valid preset.args.validate() + options = preset.args.parsed # print help if no arguments if len(sys.argv) == 1: @@ -37,25 +40,25 @@ async def _main(): return # --version - if preset.args.parsed.version: + if options.version: log.stdout(__version__) sys.exit(0) return # --current-preset - if preset.args.parsed.current_preset: + if options.current_preset: log.stdout(preset.to_yaml()) sys.exit(0) return # --current-preset-full - if preset.args.parsed.current_preset_full: + if options.current_preset_full: log.stdout(preset.to_yaml(full_config=True)) sys.exit(0) return # --list-modules - if preset.args.parsed.list_modules: + if options.list_modules: log.stdout("") log.stdout("### MODULES ###") log.stdout("") @@ -64,7 +67,7 @@ async def _main(): return # --list-flags - if preset.args.parsed.list_flags: + if options.list_flags: log.stdout("") log.stdout("### FLAGS ###") log.stdout("") @@ -73,12 +76,14 @@ async def _main(): return scan = Scanner(preset=preset) + scan_name = str(scan.name) + await scan._prep() - if not preset.args.parsed.dry_run: + if not options.dry_run: log.trace(f"Command: {' '.join(sys.argv)}") if sys.stdin.isatty(): - if not preset.args.parsed.agent_mode and not preset.args.parsed.yes: + if not options.agent_mode and not options.yes: log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") input() diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml new file mode 100644 index 000000000..5fcf0876e --- /dev/null +++ b/bbot/presets/subdomain-enum.yml @@ -0,0 +1,5 @@ +flags: + - subdomain-enum + +output_modules: + - subdomains From d35785101462a797c31c9d6624479e6289edc35b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 18 Mar 2024 07:35:35 -0400 Subject: [PATCH 045/171] remove agent functionality --- bbot/agent/__init__.py | 1 - bbot/agent/agent.py | 204 ---------------------------- bbot/agent/messages.py | 29 ---- bbot/cli.py | 2 +- bbot/defaults.yml | 4 - bbot/scanner/preset/args.py | 2 - bbot/test/test_step_1/test_agent.py | 158 --------------------- 7 files changed, 1 insertion(+), 399 deletions(-) delete mode 100644 bbot/agent/__init__.py delete mode 100644 bbot/agent/agent.py delete mode 100644 bbot/agent/messages.py delete mode 100644 bbot/test/test_step_1/test_agent.py diff --git a/bbot/agent/__init__.py b/bbot/agent/__init__.py deleted file mode 100644 index d2361b7a3..000000000 --- a/bbot/agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .agent import Agent diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py deleted file mode 100644 index 03e318683..000000000 --- a/bbot/agent/agent.py +++ /dev/null @@ -1,204 +0,0 @@ -import json -import asyncio -import logging -import traceback -import websockets -from omegaconf import OmegaConf - -from . import messages -import bbot.core.errors -from bbot.scanner import Scanner -from bbot.scanner.dispatcher import Dispatcher -from bbot.core.helpers.misc import urlparse, split_host_port -from bbot.core.configurator.environ import prepare_environment - -log = logging.getLogger("bbot.core.agent") - - -class Agent: - def __init__(self, config): - self.config = config - prepare_environment(self.config) - self.url = self.config.get("agent_url", "") - self.parsed_url = urlparse(self.url) - self.host, self.port = split_host_port(self.parsed_url.netloc) - self.token = self.config.get("agent_token", "") - self.scan = None - self.task = None - self._ws = None - self._scan_lock = asyncio.Lock() - - self.dispatcher = Dispatcher() - self.dispatcher.on_status = self.on_scan_status - self.dispatcher.on_finish = self.on_scan_finish - - def setup(self): - if not self.url: - log.error(f"Must specify agent_url") - return False - if not self.token: - log.error(f"Must specify agent_token") - return False - return True - - async def ws(self, rebuild=False): - if self._ws is None or rebuild: - kwargs = {"close_timeout": 0.5} - if self.token: - kwargs.update({"extra_headers": {"Authorization": f"Bearer {self.token}"}}) - verbs = ("Building", "Built") - if rebuild: - verbs = ("Rebuilding", "Rebuilt") - url = f"{self.url}/control/" - log.debug(f"{verbs[0]} websocket connection to {url}") - while 1: - try: - self._ws = await websockets.connect(url, **kwargs) - break - except Exception as e: - log.error(f'Failed to establish websockets connection to URL "{url}": {e}') - log.trace(traceback.format_exc()) - await asyncio.sleep(1) - log.debug(f"{verbs[1]} websocket connection to {url}") - return self._ws - - async def start(self): - rebuild = False - while 1: - ws = await self.ws(rebuild=rebuild) - rebuild = False - try: - message = await ws.recv() - log.debug(f"Got message: {message}") - try: - message = json.loads(message) - message = messages.Message(**message) - - if message.command == "ping": - if self.scan is None: - await self.send({"conversation": str(message.conversation), "message_type": "pong"}) - continue - - command_type = getattr(messages, message.command, None) - if command_type is None: - log.warning(f'Invalid command: "{message.command}"') - continue - - command_args = command_type(**message.arguments) - command_fn = getattr(self, message.command) - response = await self.err_handle(command_fn, **command_args.dict()) - log.info(str(response)) - await self.send({"conversation": str(message.conversation), "message": response}) - - except json.decoder.JSONDecodeError as e: - log.warning(f'Failed to decode message "{message}": {e}') - log.trace(traceback.format_exc()) - continue - except Exception as e: - log.debug(f"Error receiving message: {e}") - log.debug(traceback.format_exc()) - await asyncio.sleep(1) - rebuild = True - - async def send(self, message): - rebuild = False - while 1: - try: - ws = await self.ws(rebuild=rebuild) - j = json.dumps(message) - log.debug(f"Sending message of length {len(message)}") - await ws.send(j) - rebuild = False - break - except Exception as e: - log.warning(f"Error sending message: {e}, retrying") - log.trace(traceback.format_exc()) - await asyncio.sleep(1) - # rebuild = True - - async def start_scan(self, scan_id, name=None, targets=[], modules=[], output_modules=[], config={}): - async with self._scan_lock: - if self.scan is None: - log.success( - f"Starting scan with targets={targets}, modules={modules}, output_modules={output_modules}" - ) - output_module_config = OmegaConf.create( - {"modules": {"websocket": {"url": f"{self.url}/scan/{scan_id}/", "token": self.token}}} - ) - config = OmegaConf.create(config) - config = OmegaConf.merge(self.config, config, output_module_config) - output_modules = list(set(output_modules + ["websocket"])) - scan = Scanner( - *targets, - scan_id=scan_id, - name=name, - modules=modules, - output_modules=output_modules, - config=config, - dispatcher=self.dispatcher, - ) - self.task = asyncio.create_task(self._start_scan_task(scan)) - - return {"success": f"Started scan", "scan_id": scan.id} - else: - msg = f"Scan {self.scan.id} already in progress" - log.warning(msg) - return {"error": msg, "scan_id": self.scan.id} - - async def _start_scan_task(self, scan): - self.scan = scan - try: - await scan.async_start_without_generator() - except bbot.core.errors.ScanError as e: - log.error(f"Scan error: {e}") - log.trace(traceback.format_exc()) - except Exception: - log.critical(f"Encountered error: {traceback.format_exc()}") - self.on_scan_status("FAILED", scan.id) - finally: - self.task = None - - async def stop_scan(self): - log.warning("Stopping scan") - try: - async with self._scan_lock: - if self.scan is None: - msg = "Scan not in progress" - log.warning(msg) - return {"error": msg} - scan_id = str(self.scan.id) - self.scan.stop() - msg = f"Stopped scan {scan_id}" - log.warning(msg) - self.scan = None - return {"success": msg, "scan_id": scan_id} - except Exception as e: - log.warning(f"Error while stopping scan: {e}") - log.trace(traceback.format_exc()) - finally: - self.scan = None - self.task = None - - async def scan_status(self): - async with self._scan_lock: - if self.scan is None: - msg = "Scan not in progress" - log.warning(msg) - return {"error": msg} - return {"success": "Polled scan", "scan_status": self.scan.status} - - async def on_scan_status(self, status, scan_id): - await self.send({"message_type": "scan_status_change", "status": str(status), "scan_id": scan_id}) - - async def on_scan_finish(self, scan): - self.scan = None - self.task = None - - async def err_handle(self, callback, *args, **kwargs): - try: - return await callback(*args, **kwargs) - except Exception as e: - msg = f"Error in {callback.__qualname__}(): {e}" - log.error(msg) - log.trace(traceback.format_exc()) - return {"error": msg} diff --git a/bbot/agent/messages.py b/bbot/agent/messages.py deleted file mode 100644 index 34fd2c15c..000000000 --- a/bbot/agent/messages.py +++ /dev/null @@ -1,29 +0,0 @@ -from uuid import UUID -from typing import Optional -from pydantic import BaseModel - - -class Message(BaseModel): - conversation: UUID - command: str - arguments: Optional[dict] = {} - - -### COMMANDS ### - - -class start_scan(BaseModel): - scan_id: str - targets: list - modules: list - output_modules: list = [] - config: dict = {} - name: Optional[str] = None - - -class stop_scan(BaseModel): - pass - - -class scan_status(BaseModel): - pass diff --git a/bbot/cli.py b/bbot/cli.py index d3eab1b14..ebf7a8cee 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -83,7 +83,7 @@ async def _main(): log.trace(f"Command: {' '.join(sys.argv)}") if sys.stdin.isatty(): - if not options.agent_mode and not options.yes: + if not options.yes: log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") input() diff --git a/bbot/defaults.yml b/bbot/defaults.yml index d29d6ecbb..6255f0b71 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -134,10 +134,6 @@ omit_event_types: - URL_UNVERIFIED - DNS_NAME_UNRESOLVED # - IP_ADDRESS -# URL of BBOT server -agent_url: '' -# Agent Bearer authentication token -agent_token: '' # Custom interactsh server settings interactsh_server: null diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 7c69fd386..6967b0184 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -267,8 +267,6 @@ def create_parser(self, *args, **kwargs): "--ignore-failed-deps", action="store_true", help="Run modules even if they have failed dependencies" ) g2.add_argument("--install-all-deps", action="store_true", help="Install dependencies for all modules") - agent = p.add_argument_group(title="Agent", description="Report back to a central server") - agent.add_argument("-a", "--agent-mode", action="store_true", help="Start in agent mode") misc = p.add_argument_group(title="Misc") misc.add_argument("--version", action="store_true", help="show BBOT version and exit") return p diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py deleted file mode 100644 index 00d70a751..000000000 --- a/bbot/test/test_step_1/test_agent.py +++ /dev/null @@ -1,158 +0,0 @@ -import json -import websockets -from functools import partial - -from ..bbot_fixtures import * # noqa: F401 - - -_first_run = True -success = False - - -async def websocket_handler(websocket, path, scan_done=None): - # whether this is the first run - global _first_run - first_run = int(_first_run) - # whether the test succeeded - global success - # test phase - phase = "ping" - # control channel or event channel? - control = True - - if path == "/control/" and first_run: - # test ping - await websocket.send(json.dumps({"conversation": "90196cc1-299f-4555-82a0-bc22a4247590", "command": "ping"})) - _first_run = False - else: - control = False - - # Bearer token - assert websocket.request_headers["Authorization"] == "Bearer test" - - async for message in websocket: - log.debug(f"PHASE: {phase}, MESSAGE: {message}") - if not control or not first_run: - continue - m = json.loads(message) - # ping - if phase == "ping": - assert json.loads(message)["message_type"] == "pong" - phase = "start_scan_bad" - if phase == "start_scan_bad": - await websocket.send( - json.dumps( - { - "conversation": "90196cc1-299f-4555-82a0-bc22a4247590", - "command": "start_scan", - "arguments": { - "scan_id": "90196cc1-299f-4555-82a0-bc22a4247590", - "targets": ["127.0.0.2"], - "modules": ["asdf"], - "output_modules": ["human"], - "name": "agent_test_scan_bad", - }, - } - ) - ) - phase = "success" - continue - # scan start success - if phase == "success": - assert m["message"]["success"] == "Started scan" - phase = "cleaning_up" - continue - # CLEANING_UP status message - if phase == "cleaning_up": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "CLEANING_UP" - phase = "failed" - continue - # FAILED status message - if phase == "failed": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "FAILED" - phase = "start_scan" - # start good scan - if phase == "start_scan": - await websocket.send( - json.dumps( - { - "conversation": "90196cc1-299f-4555-82a0-bc22a4247590", - "command": "start_scan", - "arguments": { - "scan_id": "90196cc1-299f-4555-82a0-bc22a4247590", - "targets": ["127.0.0.2"], - "modules": ["ipneighbor"], - "output_modules": ["human"], - "name": "agent_test_scan", - }, - } - ) - ) - phase = "success_2" - continue - # scan start success - if phase == "success_2": - assert m["message"]["success"] == "Started scan" - phase = "starting" - continue - # STARTING status message - if phase == "starting": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "STARTING" - phase = "running" - continue - # RUNNING status message - if phase == "running": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "RUNNING" - phase = "finishing" - continue - # FINISHING status message - if phase == "finishing": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "FINISHING" - phase = "cleaning_up_2" - continue - # CLEANING_UP status message - if phase == "cleaning_up_2": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "CLEANING_UP" - phase = "finished_2" - continue - # FINISHED status message - if phase == "finished_2": - assert m["message_type"] == "scan_status_change" - assert m["status"] == "FINISHED" - success = True - scan_done.set() - break - - -@pytest.mark.asyncio -async def test_agent(agent): - scan_done = asyncio.Event() - scan_status = await agent.scan_status() - assert scan_status["error"] == "Scan not in progress" - - _websocket_handler = partial(websocket_handler, scan_done=scan_done) - - global success - async with websockets.serve(_websocket_handler, "127.0.0.1", 8765): - agent_task = asyncio.create_task(agent.start()) - # wait for 90 seconds - await asyncio.wait_for(scan_done.wait(), 60) - assert success - - await agent.start_scan("scan_to_be_cancelled", targets=["127.0.0.1"], modules=["ipneighbor"]) - await agent.start_scan("scan_to_be_rejected", targets=["127.0.0.1"], modules=["ipneighbor"]) - await asyncio.sleep(0.1) - await agent.stop_scan() - tasks = [agent.task, agent_task] - for task in tasks: - try: - task.cancel() - await task - except (asyncio.CancelledError, AttributeError): - pass From fd822118cef0d08281afb17ab317e06c73ccb97e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 18 Mar 2024 15:17:43 -0400 Subject: [PATCH 046/171] recursive preset inclusion feature --- bbot/presets/email-enum.yml | 24 +++++++++++++ bbot/presets/spider.yml | 24 +++++++++++++ bbot/presets/subdomain-enum.yml | 15 +++++++++ bbot/scanner/preset/args.py | 8 +++-- bbot/scanner/preset/path.py | 13 ++++--- bbot/scanner/preset/preset.py | 60 +++++++++++++++++++-------------- 6 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 bbot/presets/email-enum.yml create mode 100644 bbot/presets/spider.yml diff --git a/bbot/presets/email-enum.yml b/bbot/presets/email-enum.yml new file mode 100644 index 000000000..5eba4f081 --- /dev/null +++ b/bbot/presets/email-enum.yml @@ -0,0 +1,24 @@ +# include web spider (emails are automatically extracted from HTTP_RESPONSEs) +include: + - spider + +flags: + - email-enum + +output_modules: + - emails + +config: + modules: + stdout: + format: text + # only output EMAIL_ADDRESSes to the console + event_types: + - EMAIL_ADDRESS + # only show in-scope emails + in_scope_only: True + # display the raw emails, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False diff --git a/bbot/presets/spider.yml b/bbot/presets/spider.yml new file mode 100644 index 000000000..d868b76e3 --- /dev/null +++ b/bbot/presets/spider.yml @@ -0,0 +1,24 @@ +modules: + - httpx + +config: + # how many links to follow in a row + web_spider_distance: 2 + # don't follow links whose directory depth is higher than 3 + web_spider_depth: 3 + # maximum number of links to follow per page + web_spider_links_per_page: 25 + + modules: + stdout: + format: text + # only output URLs to the console + event_types: + - URL + # only show in-scope URLs + in_scope_only: True + # display the raw URLs, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index 5fcf0876e..8310ed158 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -3,3 +3,18 @@ flags: output_modules: - subdomains + +config: + modules: + stdout: + format: text + # only output DNS_NAMEs to the console + event_types: + - DNS_NAME + # only show in-scope subdomains + in_scope_only: True + # display the raw subdomains, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 6967b0184..625416c0f 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -2,10 +2,10 @@ import sys import logging import argparse +import traceback from omegaconf import OmegaConf from bbot.core.errors import PresetNotFoundError -from bbot.core.helpers.logger import log_to_stderr from bbot.core.helpers.misc import chain_lists, match_and_exit log = logging.getLogger("bbot.presets.args") @@ -131,10 +131,12 @@ def preset_from_args(self): preset_args.append(preset_param) args_preset.core.merge_custom(cli_config) except Exception as e: - log_to_stderr(f"Error parsing command-line config: {e}", level="ERROR") + log.error(f"Error parsing command-line config: {e}") + log.trace(traceback.format_exc()) sys.exit(2) except Exception as e: - log_to_stderr(f"Error parsing custom config at {preset_param}: {e}", level="ERROR") + log.error(f"Error parsing custom config at {preset_param}: {e}") + log.trace(traceback.format_exc()) sys.exit(2) self.parsed.preset = preset_args diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index a382f206e..669b321f7 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -20,17 +20,19 @@ def __init__(self): def find(self, filename): filename = Path(filename) self.add_path(filename.parent) - self.add_path(filename.parent / "presets") if filename.is_file(): log.hugesuccess(filename) return filename extension = filename.suffix.lower() file_candidates = set() + if not extension: + file_candidates.add(filename.stem) for ext in (".yaml", ".yml"): if extension != ext: file_candidates.add(f"{filename.stem}{ext}") file_candidates = sorted(file_candidates) - log.debug(f"Searching for preset in {self.paths}, file candidates: {file_candidates}") + file_candidates_str = ",".join([str(s) for s in file_candidates]) + log.debug(f"Searching for preset in {self}, file candidates: {file_candidates_str}") for path in self.paths: for candidate in file_candidates: for file in path.rglob(candidate): @@ -38,12 +40,15 @@ def find(self, filename): return file raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') + def __str__(self): + return ":".join([str(s) for s in self.paths]) + def add_path(self, path): - path = Path(path) + path = Path(path).resolve() if path in self.paths: return if not path.is_dir(): - log.debug(f'Path "{path}" is not a directory') + log.debug(f'Path "{path.resolve()}" is not a directory') return self.paths.append(path) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index af73f2fac..744166b7a 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -36,6 +36,7 @@ def __init__( config=None, strict_scope=False, module_dirs=None, + include_presets=None, ): self._args = None self._environ = None @@ -53,10 +54,6 @@ def __init__( self._debug = False self._silent = False - # custom module directories - self._module_dirs = set() - self.module_dirs = module_dirs - # bbot core config self.core = CORE.copy() if config is None: @@ -64,6 +61,38 @@ def __init__( # merge any custom configs self.core.merge_custom(config) + # log verbosity + if verbose: + self.verbose = verbose + if debug: + self.debug = debug + if silent: + self.silent = silent + + # custom module directories + self._module_dirs = set() + self.module_dirs = module_dirs + + self.strict_scope = strict_scope + + # target / whitelist / blacklist + from bbot.scanner.target import Target + + self.target = Target(*targets, strict_scope=self.strict_scope) + if not whitelist: + self.whitelist = self.target.copy() + else: + self.whitelist = Target(*whitelist, strict_scope=self.strict_scope) + if not blacklist: + blacklist = [] + self.blacklist = Target(*blacklist) + + # include other presets + if include_presets: + for preset in include_presets: + log.debug(f'Including preset "{preset}"') + self.merge(self.from_yaml_file(preset)) + # modules + flags if modules is None: modules = [] @@ -83,28 +112,6 @@ def __init__( self.output_modules = output_modules if output_modules is not None else [] self.internal_modules = internal_modules if internal_modules is not None else [] - self.strict_scope = strict_scope - - # target / whitelist / blacklist - from bbot.scanner.target import Target - - self.target = Target(*targets, strict_scope=self.strict_scope) - if not whitelist: - self.whitelist = self.target.copy() - else: - self.whitelist = Target(*whitelist, strict_scope=self.strict_scope) - if not blacklist: - blacklist = [] - self.blacklist = Target(*blacklist) - - # log verbosity - if verbose: - self.verbose = verbose - if debug: - self.debug = debug - if silent: - self.silent = silent - @property def bbot_home(self): return Path(self.config.get("home", "~/.bbot")).expanduser().resolve() @@ -490,6 +497,7 @@ def from_dict(cls, preset_dict): config=preset_dict.get("config"), strict_scope=preset_dict.get("strict_scope", False), module_dirs=preset_dict.get("module_dirs", []), + include_presets=preset_dict.get("include", []), ) return new_preset From 1ab42770765632e660505ec5c328cfdbd1cca76d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 19 Mar 2024 15:00:08 -0400 Subject: [PATCH 047/171] social dependency --- bbot/core/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 556bfab79..995c70c92 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -37,7 +37,7 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") # if a module consumes these event types, automatically assume these dependencies - default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx"} + default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx", "SOCIAL": "social"} def __init__(self): self.core = CORE From ec2b35cfda293bd09a98c0a4172884772ab7c741 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 20 Mar 2024 11:20:22 -0400 Subject: [PATCH 048/171] recursive inclusion, circular inclusion tests --- bbot/scanner/preset/path.py | 4 +- bbot/scanner/preset/preset.py | 44 +++++++++++---- bbot/test/bbot_fixtures.py | 6 ++ bbot/test/test_step_1/test_presets.py | 64 ++++++++++++++++++++-- bbot/test/test_step_2/module_tests/base.py | 8 +-- 5 files changed, 103 insertions(+), 23 deletions(-) diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index 669b321f7..7c89dc648 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -18,7 +18,7 @@ def __init__(self): self.add_path(Path.cwd()) def find(self, filename): - filename = Path(filename) + filename = Path(filename).resolve() self.add_path(filename.parent) if filename.is_file(): log.hugesuccess(filename) @@ -37,7 +37,7 @@ def find(self, filename): for candidate in file_candidates: for file in path.rglob(candidate): log.verbose(f'Found preset matching "{filename}" at {file}') - return file + return file.resolve() raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') def __str__(self): diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 744166b7a..c11503158 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -36,7 +36,8 @@ def __init__( config=None, strict_scope=False, module_dirs=None, - include_presets=None, + include=None, + _exclude=None, ): self._args = None self._environ = None @@ -54,6 +55,11 @@ def __init__( self._debug = False self._silent = False + self._preset_files_loaded = set() + if _exclude is not None: + for _filename in _exclude: + self._preset_files_loaded.add(Path(_filename).resolve()) + # bbot core config self.core = CORE.copy() if config is None: @@ -88,10 +94,9 @@ def __init__( self.blacklist = Target(*blacklist) # include other presets - if include_presets: - for preset in include_presets: - log.debug(f'Including preset "{preset}"') - self.merge(self.from_yaml_file(preset)) + if include: + for included_preset in include: + self.include_preset(included_preset) # modules + flags if modules is None: @@ -480,7 +485,7 @@ def whitelisted(self, e): return e in self.whitelist @classmethod - def from_dict(cls, preset_dict): + def from_dict(cls, preset_dict, _exclude=None): new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -497,19 +502,36 @@ def from_dict(cls, preset_dict): config=preset_dict.get("config"), strict_scope=preset_dict.get("strict_scope", False), module_dirs=preset_dict.get("module_dirs", []), - include_presets=preset_dict.get("include", []), + include=preset_dict.get("include", []), + _exclude=_exclude, ) return new_preset + def include_preset(self, filename): + log.debug(f'Including preset "{filename}"') + preset_filename = PRESET_PATH.find(filename) + preset_from_yaml = self.from_yaml_file(preset_filename, _exclude=self._preset_files_loaded) + if preset_from_yaml is not False: + self.merge(preset_from_yaml) + self._preset_files_loaded.add(preset_filename) + @classmethod - def from_yaml_file(cls, filename): + def from_yaml_file(cls, filename, _exclude=None): """ Create a preset from a YAML file. If the full path is not specified, BBOT will look in all the usual places for it. - Specifying the file extension is optional. + The file extension is optional. """ - yaml_preset = PRESET_PATH.find(filename) - return cls.from_dict(omegaconf.OmegaConf.load(yaml_preset)) + if _exclude is None: + _exclude = set() + filename = Path(filename).resolve() + if _exclude is not None and filename in _exclude: + log.debug(f"Not merging {filename} because it was already loaded {_exclude}") + return False + log.debug(f"Merging {filename} because it's not in {_exclude}") + _exclude = set(_exclude) + _exclude.add(filename) + return cls.from_dict(omegaconf.OmegaConf.load(filename), _exclude=_exclude) @classmethod def from_yaml_string(cls, yaml_preset): diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 8cbdd0922..17a44ee51 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -12,6 +12,12 @@ from werkzeug.wrappers import Request +from bbot.core.helpers.misc import mkdir + + +bbot_test_dir = Path("/tmp/.bbot_test") +mkdir(bbot_test_dir) + class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): def match_data(self, request: Request) -> bool: diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index e7dd99452..94807dcfb 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -305,10 +305,7 @@ def test_preset_module_resolution(): def test_preset_module_loader(): - - from pathlib import Path - - custom_module_dir = Path("/tmp/.bbot_test/custom_module_dir") + custom_module_dir = bbot_test_dir / "custom_module_dir" custom_module_dir_2 = custom_module_dir / "asdf" custom_output_module_dir = custom_module_dir / "output" custom_internal_module_dir = custom_module_dir / "internal" @@ -413,7 +410,64 @@ class TestModule4(BaseModule): assert "testmodule4" in preset2.module_loader.preloaded() -# test recursive include +def test_preset_include(): + + # test recursive preset inclusion + + custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir" + custom_preset_dir_2 = custom_preset_dir_1 / "preset_subdir" + custom_preset_dir_3 = custom_preset_dir_2 / "subsubdir" + mkdir(custom_preset_dir_1) + mkdir(custom_preset_dir_2) + mkdir(custom_preset_dir_3) + + preset_file = custom_preset_dir_1 / "preset1.yml" + with open(preset_file, "w") as f: + f.write( + """ +include: + - preset2 + +config: + modules: + testpreset1: + test: asdf +""" + ) + + preset_file = custom_preset_dir_2 / "preset2.yml" + with open(preset_file, "w") as f: + f.write( + """ +include: + - preset3 + +config: + modules: + testpreset2: + test: fdsa +""" + ) + + preset_file = custom_preset_dir_3 / "preset3.yml" + with open(preset_file, "w") as f: + f.write( + """ +include: + # uh oh + - preset1 + +config: + modules: + testpreset3: + test: qwerty +""" + ) + + preset = Preset(include=[custom_preset_dir_1 / "preset1"]) + assert preset.config.modules.testpreset1.test == "asdf" + assert preset.config.modules.testpreset2.test == "fdsa" + assert preset.config.modules.testpreset3.test == "qwerty" # test custom module load directory diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 73c37dec0..985dc5c79 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -8,17 +8,15 @@ from bbot.scanner import Scanner from bbot.core import CORE from bbot.core.helpers.misc import rand_string -from ...bbot_fixtures import test_config, MockResolver +from ...bbot_fixtures import test_config, MockResolver, bbot_test_dir log = logging.getLogger("bbot.test.modules") def tempwordlist(content): - tmp_path = "/tmp/.bbot_test/" - from bbot.core.helpers.misc import rand_string, mkdir + from bbot.core.helpers.misc import rand_string - mkdir(tmp_path) - filename = f"{tmp_path}{rand_string(8)}" + filename = bbot_test_dir / f"{rand_string(8)}" with open(filename, "w", errors="ignore") as f: for c in content: line = f"{c}\n" From 40ff3baefd04e49f47b5fb7e3fa2f32c81a6ae9d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 20 Mar 2024 14:23:44 -0400 Subject: [PATCH 049/171] cli arg tests passing --- bbot/cli.py | 175 ++++++++++++++++---------- bbot/core/errors.py | 8 ++ bbot/core/helpers/wordcloud.py | 2 +- bbot/core/modules.py | 10 +- bbot/scanner/preset/args.py | 9 +- bbot/scanner/preset/preset.py | 56 ++++++--- bbot/scanner/scanner.py | 22 ++-- bbot/test/bbot_fixtures.py | 3 + bbot/test/test_step_1/test_cli.py | 102 +++++++++------ bbot/test/test_step_1/test_presets.py | 3 + 10 files changed, 259 insertions(+), 131 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index ebf7a8cee..08f77f504 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -4,6 +4,7 @@ import asyncio import logging import traceback +from contextlib import suppress # fix tee buffering sys.stdout.reconfigure(line_buffering=True) @@ -25,69 +26,117 @@ async def _main(): global scan_name - # start by creating a default scan preset - preset = Preset() - # parse command line arguments and merge into preset - preset.parse_args() - # ensure arguments (-c config options etc.) are valid - preset.args.validate() - options = preset.args.parsed - - # print help if no arguments - if len(sys.argv) == 1: - preset.args.parser.print_help() - sys.exit(1) - return - - # --version - if options.version: - log.stdout(__version__) - sys.exit(0) - return - - # --current-preset - if options.current_preset: - log.stdout(preset.to_yaml()) - sys.exit(0) - return - - # --current-preset-full - if options.current_preset_full: - log.stdout(preset.to_yaml(full_config=True)) - sys.exit(0) - return - - # --list-modules - if options.list_modules: - log.stdout("") - log.stdout("### MODULES ###") - log.stdout("") - for row in preset.module_loader.modules_table(preset.modules).splitlines(): - log.stdout(row) - return - - # --list-flags - if options.list_flags: - log.stdout("") - log.stdout("### FLAGS ###") - log.stdout("") - for row in preset.module_loader.flags_table(flags=preset.flags).splitlines(): - log.stdout(row) - return - - scan = Scanner(preset=preset) - scan_name = str(scan.name) - await scan._prep() - - if not options.dry_run: - log.trace(f"Command: {' '.join(sys.argv)}") - - if sys.stdin.isatty(): - if not options.yes: - log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") - input() - - await scan.async_start_without_generator() + try: + + # start by creating a default scan preset + preset = Preset() + # parse command line arguments and merge into preset + preset.parse_args() + # ensure arguments (-c config options etc.) are valid + options = preset.args.parsed + + modules_to_list = None + if options.modules or options.output_modules: + modules_to_list = set() + if options.modules: + modules_to_list.update(set(preset.scan_modules)) + if options.output_modules: + modules_to_list.update(set(preset.output_modules)) + + # print help if no arguments + if len(sys.argv) == 1: + preset.args.parser.print_help() + sys.exit(1) + return + + # --version + if options.version: + log.stdout(__version__) + sys.exit(0) + return + + # --current-preset + if options.current_preset: + log.stdout(preset.to_yaml()) + sys.exit(0) + return + + # --current-preset-full + if options.current_preset_full: + log.stdout(preset.to_yaml(full_config=True)) + sys.exit(0) + return + + # --list-modules + if options.list_modules: + log.stdout("") + log.stdout("### MODULES ###") + log.stdout("") + for row in preset.module_loader.modules_table(modules_to_list).splitlines(): + log.stdout(row) + return + + # --list-module-options + if options.list_module_options: + log.stdout("") + log.stdout("### MODULE OPTIONS ###") + log.stdout("") + for row in preset.module_loader.modules_options_table(modules=modules_to_list).splitlines(): + log.stdout(row) + return + + # --list-flags + if options.list_flags: + flags = preset.flags if preset.flags else None + log.stdout("") + log.stdout("### FLAGS ###") + log.stdout("") + for row in preset.module_loader.flags_table(flags=flags).splitlines(): + log.stdout(row) + return + + deadly_modules = [m for m in preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", [])] + if deadly_modules and not options.allow_deadly: + log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") + log.hugewarning(f"Deadly modules are highly intrusive") + log.hugewarning(f"Please specify --allow-deadly to continue") + return False + + scan = Scanner(preset=preset) + + # --install-all-deps + if options.install_all_deps: + all_modules = list(preset.module_loader.preloaded()) + scan.helpers.depsinstaller.force_deps = True + succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules) + log.info("Finished installing module dependencies") + return False if failed else True + + scan_name = str(scan.name) + scan.helpers.word_cloud.load() + await scan._prep() + + if not options.dry_run: + log.trace(f"Command: {' '.join(sys.argv)}") + + if sys.stdin.isatty(): + if not options.yes: + log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") + input() + + await scan.async_start_without_generator() + + return True + + finally: + # save word cloud + with suppress(BaseException): + save_success, filename = scan.helpers.word_cloud.save() + if save_success: + log_to_stderr(f"Saved word cloud ({len(scan.helpers.word_cloud):,} words) to {filename}") + # remove output directory if empty + with suppress(BaseException): + scan.home.rmdir() def main(): diff --git a/bbot/core/errors.py b/bbot/core/errors.py index ce4e3dcc9..6f13c9875 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -52,3 +52,11 @@ class CurlError(BBOTError): class PresetNotFoundError(BBOTError): pass + + +class EnableModuleError(BBOTError): + pass + + +class EnableFlagError(BBOTError): + pass diff --git a/bbot/core/helpers/wordcloud.py b/bbot/core/helpers/wordcloud.py index 26d050406..5eafb00c5 100644 --- a/bbot/core/helpers/wordcloud.py +++ b/bbot/core/helpers/wordcloud.py @@ -322,7 +322,7 @@ def json(self, limit=None): @property def default_filename(self): - return self.parent_helper.scan.home / f"wordcloud.tsv" + return self.parent_helper.preset.scan.home / f"wordcloud.tsv" def save(self, filename=None, limit=None): """ diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 995c70c92..fe284fedb 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -45,6 +45,7 @@ def __init__(self): self.__preloaded = {} self._modules = {} self._configs = {} + self._all_flags = set() self._preload_cache = None @@ -124,7 +125,10 @@ def preload(self, module_dirs=None): # try to load from cache module_cache_key = (str(module_file), tuple(module_file.stat())) - cache_key = self.preload_cache.get(module_name, {}).get("cache_key", ()) + preloaded_module = self.preload_cache.get(module_name, {}) + flags = preloaded_module.get("flags", []) + self._all_flags.update(set(flags)) + cache_key = preloaded_module.get("cache_key", ()) if module_cache_key == cache_key: preloaded = self.preload_cache[module_name] else: @@ -138,7 +142,9 @@ def preload(self, module_dirs=None): if module_dir.name in ("output", "internal"): module_type = str(module_dir.name) elif module_dir.name not in ("modules"): - preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) + flags = set(preloaded["flags"] + [module_dir.name]) + self._all_flags.update(flags) + preloaded["flags"] = sorted(flags) # derive module dependencies from watched event types (only for scan modules) if module_type == "scan": for event_type in preloaded["watched_events"]: diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 625416c0f..cd0f3d292 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -90,6 +90,7 @@ def parsed(self): return self._parsed def preset_from_args(self): + self.validate() args_preset = self.preset.__class__( *self.parsed.targets, whitelist=self.parsed.whitelist, @@ -151,6 +152,10 @@ def preset_from_args(self): elif self.parsed.ignore_failed_deps: args_preset.core.custom_config["deps_behavior"] = "ignore_failed" + # other scan options + args_preset.scan_name = self.parsed.name + args_preset.output_dir = self.parsed.output_dir + return args_preset def create_parser(self, *args, **kwargs): @@ -160,7 +165,6 @@ def create_parser(self, *args, **kwargs): ) ) p = argparse.ArgumentParser(*args, **kwargs) - p.add_argument("--help-all", action="store_true", help="Display full help including module config options") target = p.add_argument_group(title="Target") target.add_argument( "-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET" @@ -188,6 +192,9 @@ def create_parser(self, *args, **kwargs): metavar="MODULE", ) modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") + modules.add_argument( + "-lmo", "--list-module-options", action="store_true", help="Show all module config options" + ) modules.add_argument( "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" ) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index c11503158..d4f06dc9b 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -10,7 +10,7 @@ from bbot.core import CORE from bbot.core.event.base import make_event -from bbot.core.errors import ValidationError +from bbot.core.errors import EnableFlagError, EnableModuleError, PresetNotFoundError, ValidationError log = logging.getLogger("bbot.presets") @@ -37,8 +37,11 @@ def __init__( strict_scope=False, module_dirs=None, include=None, + output_dir=None, + scan_name=None, _exclude=None, ): + self.scan = None self._args = None self._environ = None self._helpers = None @@ -55,6 +58,9 @@ def __init__( self._debug = False self._silent = False + self.output_dir = output_dir + self.scan_name = scan_name + self._preset_files_loaded = set() if _exclude is not None: for _filename in _exclude: @@ -153,6 +159,11 @@ def merge(self, other): self.verbose = other.verbose if other.debug: self.debug = other.debug + # scan name + if other.scan_name is not None: + self.scan_name = other.scan_name + if other.output_dir is not None: + self.output_dir = other.output_dir def bake(self): """ @@ -248,26 +259,27 @@ def add_module(self, module_name, module_type="scan"): try: preloaded = self.module_loader.preloaded()[module_name] except KeyError: - raise KeyError(f'Unable to add unknown BBOT module "{module_name}"') + raise EnableModuleError(f'Unable to add unknown BBOT module "{module_name}"') module_flags = preloaded.get("flags", []) + _module_type = preloaded.get("type", "scan") if module_type: - _module_type = preloaded.get("type", "scan") if _module_type != module_type: log.verbose( f'Not enabling module "{module_name}" because its type ({_module_type}) is not "{module_type}"' ) return - for f in module_flags: - if f in self.exclude_flags: - log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + if _module_type == "scan": + for f in module_flags: + if f in self.exclude_flags: + log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + return + if self.require_flags and not any(f in self.require_flags for f in module_flags): + log.verbose( + f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' + ) return - if self.require_flags and not any(f in self.require_flags for f in module_flags): - log.verbose( - f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' - ) - return if module_name not in self.modules: log.verbose(f'Enabling module "{module_name}"') @@ -297,6 +309,9 @@ def flags(self): def flags(self, flags): if isinstance(flags, str): flags = [flags] + for flag in flags: + if not flag in self.module_loader._all_flags: + raise EnableFlagError(f'Flag "{flag}" was not found') self._flags = set(flags) if self._flags: for module, preloaded in self.module_loader.preloaded().items(): @@ -330,7 +345,7 @@ def exclude_flags(self, flags): def add_required_flag(self, flag): self.require_flags.add(flag) - for module in list(self.modules): + for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) if flag not in module_flags: log.verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') @@ -338,7 +353,7 @@ def add_required_flag(self, flag): def add_excluded_flag(self, flag): self.exclude_flags.add(flag) - for module in list(self.modules): + for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) if flag in module_flags: log.verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') @@ -346,7 +361,7 @@ def add_excluded_flag(self, flag): def add_excluded_module(self, module): self.exclude_modules.add(module) - for module in list(self.modules): + for module in list(self.scan_modules): if module in self.exclude_modules: log.verbose(f'Removing module "{module}" because it\'s excluded') self.modules.remove(module) @@ -503,6 +518,8 @@ def from_dict(cls, preset_dict, _exclude=None): strict_scope=preset_dict.get("strict_scope", False), module_dirs=preset_dict.get("module_dirs", []), include=preset_dict.get("include", []), + scan_name=preset_dict.get("scan_name"), + output_dir=preset_dict.get("output_dir"), _exclude=_exclude, ) return new_preset @@ -531,7 +548,10 @@ def from_yaml_file(cls, filename, _exclude=None): log.debug(f"Merging {filename} because it's not in {_exclude}") _exclude = set(_exclude) _exclude.add(filename) - return cls.from_dict(omegaconf.OmegaConf.load(filename), _exclude=_exclude) + try: + return cls.from_dict(omegaconf.OmegaConf.load(filename), _exclude=_exclude) + except FileNotFoundError: + raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') @classmethod def from_yaml_string(cls, yaml_preset): @@ -587,6 +607,12 @@ def to_dict(self, include_target=False, full_config=False): if self.silent: preset_dict["silent"] = True + # misc scan options + if self.scan_name: + preset_dict["scan_name"] = self.scan_name + if self.scan_name: + preset_dict["output_dir"] = self.output_dir + return preset_dict def to_yaml(self, include_target=False, full_config=False, sort_keys=False): diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index dc32a9bda..4cb0c8a8f 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -104,8 +104,6 @@ def __init__( self, *args, scan_id=None, - scan_name=None, - output_dir=None, dispatcher=None, force_start=False, **preset_kwargs, @@ -136,28 +134,30 @@ def __init__( if preset is None: preset = Preset(*args, **preset_kwargs) self.preset = preset.bake() + self.preset.scan = self # scan name - if scan_name is None: + if preset.scan_name is None: tries = 0 while 1: if tries > 5: - self.name = f"{rand_string(4)}_{rand_string(4)}" + scan_name = f"{rand_string(4)}_{rand_string(4)}" break - self.name = random_name() - if output_dir is not None: - home_path = Path(output_dir).resolve() / self.name + scan_name = random_name() + if self.preset.output_dir is not None: + home_path = Path(self.preset.output_dir).resolve() / scan_name else: - home_path = self.preset.bbot_home / "scans" / self.name + home_path = self.preset.bbot_home / "scans" / scan_name if not home_path.exists(): break tries += 1 else: - self.name = str(scan_name) + scan_name = str(preset.scan_name) + self.name = scan_name # scan output dir - if output_dir is not None: - self.home = Path(output_dir).resolve() / self.name + if preset.output_dir is not None: + self.home = Path(preset.output_dir).resolve() / self.name else: self.home = self.preset.bbot_home / "scans" / self.name diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 17a44ee51..a451cdd45 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -12,6 +12,7 @@ from werkzeug.wrappers import Request +from bbot.core import CORE from bbot.core.helpers.misc import mkdir @@ -33,6 +34,8 @@ def match_data(self, request: Request) -> bool: if test_config.get("debug", False): os.environ["BBOT_DEBUG"] = "True" +CORE.merge_custom(test_config) + from .bbot_fixtures import * # noqa: F401 from bbot.core.errors import * # noqa: F401 diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 0ccc94887..e2d2de572 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -2,12 +2,11 @@ @pytest.mark.asyncio -async def test_cli(monkeypatch, bbot_config): +async def test_cli_args(monkeypatch, bbot_config): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - monkeypatch.setattr(cli, "config", bbot_config) old_sys_argv = sys.argv @@ -44,56 +43,90 @@ async def test_cli(monkeypatch, bbot_config): dns_success = True assert ip_success and dns_success, "IP_ADDRESS and/or DNS_NAME are not present in output.txt" + # nonexistent module + monkeypatch.setattr("sys.argv", ["bbot", "-m", "asdf"]) + with pytest.raises(EnableModuleError): + result = await cli._main() + + # nonexistent output module + monkeypatch.setattr("sys.argv", ["bbot", "-om", "asdf"]) + with pytest.raises(EnableModuleError): + result = await cli._main() + + # nonexistent flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "asdf"]) + with pytest.raises(EnableFlagError): + result = await cli._main() + # show version monkeypatch.setattr("sys.argv", ["bbot", "--version"]) - await cli._main() + result = await cli._main() + assert result == None - # start agent - monkeypatch.setattr("sys.argv", ["bbot", "--agent-mode"]) - task = asyncio.create_task(cli._main()) - await asyncio.sleep(2) - task.cancel() - try: - await task - except asyncio.CancelledError: - pass + # show current preset + monkeypatch.setattr("sys.argv", ["bbot", "--current-preset"]) + result = await cli._main() + assert result == None + + # show current preset (full) + monkeypatch.setattr("sys.argv", ["bbot", "--current-preset-full"]) + result = await cli._main() + assert result == None + + # list modules + monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"]) + result = await cli._main() + assert result == None + + # list module options + monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"]) + result = await cli._main() + assert result == None + + # list flags + monkeypatch.setattr("sys.argv", ["bbot", "--list-flags"]) + result = await cli._main() + assert result == None # no args monkeypatch.setattr("sys.argv", ["bbot"]) - await cli._main() + result = await cli._main() + assert result == None # enable module by flag monkeypatch.setattr("sys.argv", ["bbot", "-f", "report"]) - await cli._main() + result = await cli._main() + assert result == True # unconsoleable output module monkeypatch.setattr("sys.argv", ["bbot", "-om", "web_report"]) - await cli._main() - - # install all deps - monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"]) - success = await cli._main() - assert success, "--install-all-deps failed for at least one module" + result = await cli._main() + assert result == True # unresolved dependency monkeypatch.setattr("sys.argv", ["bbot", "-m", "wappalyzer"]) - await cli._main() + result = await cli._main() + assert result == True # resolved dependency, excluded module monkeypatch.setattr("sys.argv", ["bbot", "-m", "ffuf_shortnames", "-em", "ffuf_shortnames"]) - await cli._main() + result = await cli._main() + assert result == True # require flags monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "-rf", "passive"]) - await cli._main() + result = await cli._main() + assert result == True # excluded flags monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "-ef", "active"]) - await cli._main() + result = await cli._main() + assert result == True # slow modules - monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdns"]) - await cli._main() + monkeypatch.setattr("sys.argv", ["bbot", "-m", "bucket_digitalocean"]) + result = await cli._main() + assert result == True # deadly modules monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei"]) @@ -103,19 +136,12 @@ async def test_cli(monkeypatch, bbot_config): # --allow-deadly monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei", "--allow-deadly"]) result = await cli._main() - assert result != False, "-m nuclei failed to run with --allow-deadly" - - # show current config - monkeypatch.setattr("sys.argv", ["bbot", "-y", "--current-config"]) - await cli._main() - - # list modules - monkeypatch.setattr("sys.argv", ["bbot", "-l"]) - await cli._main() + assert result == True, "-m nuclei failed to run with --allow-deadly" - # list module options - monkeypatch.setattr("sys.argv", ["bbot", "--help-all"]) - await cli._main() + # install all deps + monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"]) + success = await cli._main() + assert success, "--install-all-deps failed for at least one module" # unpatch sys.argv monkeypatch.setattr("sys.argv", old_sys_argv) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 94807dcfb..c4f70e46f 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -474,3 +474,6 @@ def test_preset_include(): # make sure it works with cli arg module/flag/config syntax validation # what if you specify -c modules.custommodule.option # the validation needs to not happen until after all presets have been loaded +# what if you specify flags in one preset +# but another preset (loaded later) has more custom modules that match that flag +# what if you specify a flag that's only on custom modules? Will it be rejected as invalid? From 7acc625ed89006402295e3383dc092bd4ecb7bc6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 20 Mar 2024 16:37:49 -0400 Subject: [PATCH 050/171] more cli tests --- bbot/test/test_step_1/test_cli.py | 52 +++++++++++-------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index e2d2de572..cc40b722e 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -8,8 +8,6 @@ async def test_cli_args(monkeypatch, bbot_config): monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - old_sys_argv = sys.argv - home_dir = Path(bbot_config["home"]) scans_home = home_dir / "scans" @@ -135,81 +133,69 @@ async def test_cli_args(monkeypatch, bbot_config): # --allow-deadly monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei", "--allow-deadly"]) - result = await cli._main() - assert result == True, "-m nuclei failed to run with --allow-deadly" + # result = await cli._main() + # assert result == True, "-m nuclei failed to run with --allow-deadly" # install all deps - monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"]) - success = await cli._main() - assert success, "--install-all-deps failed for at least one module" - - # unpatch sys.argv - monkeypatch.setattr("sys.argv", old_sys_argv) + # monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"]) + # success = await cli._main() + # assert success, "--install-all-deps failed for at least one module" def test_config_validation(monkeypatch, capsys, bbot_config): from bbot import cli - from bbot.core.configurator import args monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - monkeypatch.setattr(cli, "config", bbot_config) - - old_cli_config = args.cli_config # incorrect module option - monkeypatch.setattr(args, "cli_config", ["bbot", "-c", "modules.ipnegibhor.num_bits=4"]) + monkeypatch.setattr("sys.argv", ["bbot", "-c", "modules.ipnegibhor.num_bits=4"]) cli.main() captured = capsys.readouterr() assert 'Could not find module option "modules.ipnegibhor.num_bits"' in captured.err assert 'Did you mean "modules.ipneighbor.num_bits"?' in captured.err # incorrect global option - monkeypatch.setattr(args, "cli_config", ["bbot", "-c", "web_spier_distance=4"]) + monkeypatch.setattr("sys.argv", ["bbot", "-c", "web_spier_distance=4"]) cli.main() captured = capsys.readouterr() assert 'Could not find module option "web_spier_distance"' in captured.err assert 'Did you mean "web_spider_distance"?' in captured.err - # unpatch cli_options - monkeypatch.setattr(args, "cli_config", old_cli_config) - def test_module_validation(monkeypatch, capsys, bbot_config): - from bbot.core.configurator import args + from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - old_sys_argv = sys.argv - # incorrect module - monkeypatch.setattr(sys, "argv", ["bbot", "-m", "massdnss"]) - args.parser.parse_args() + monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdnss"]) + with pytest.raises(EnableModuleError): + cli.main() captured = capsys.readouterr() assert 'Could not find module "massdnss"' in captured.err assert 'Did you mean "massdns"?' in captured.err # incorrect excluded module - monkeypatch.setattr(sys, "argv", ["bbot", "-em", "massdnss"]) - args.parser.parse_args() + monkeypatch.setattr("sys.argv", ["bbot", "-em", "massdnss"]) + cli.main() captured = capsys.readouterr() assert 'Could not find module "massdnss"' in captured.err assert 'Did you mean "massdns"?' in captured.err # incorrect output module - monkeypatch.setattr(sys, "argv", ["bbot", "-om", "neoo4j"]) - args.parser.parse_args() + monkeypatch.setattr("sys.argv", ["bbot", "-om", "neoo4j"]) + with pytest.raises(EnableModuleError): + cli.main() captured = capsys.readouterr() assert 'Could not find output module "neoo4j"' in captured.err assert 'Did you mean "neo4j"?' in captured.err # incorrect flag - monkeypatch.setattr(sys, "argv", ["bbot", "-f", "subdomainenum"]) - args.parser.parse_args() + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomainenum"]) + with pytest.raises(EnableFlagError): + cli.main() captured = capsys.readouterr() assert 'Could not find flag "subdomainenum"' in captured.err assert 'Did you mean "subdomain-enum"?' in captured.err - - # unpatch sys.argv - monkeypatch.setattr("sys.argv", old_sys_argv) From 1f2600e15991c04fffa9284296d08ba33e3a9c9e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 21 Mar 2024 10:57:08 -0400 Subject: [PATCH 051/171] better inclusion tests --- bbot/test/test_step_1/test_presets.py | 35 ++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index c4f70e46f..b47f63087 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -417,9 +417,13 @@ def test_preset_include(): custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir" custom_preset_dir_2 = custom_preset_dir_1 / "preset_subdir" custom_preset_dir_3 = custom_preset_dir_2 / "subsubdir" + custom_preset_dir_4 = Path("/tmp/.bbot_preset_test") + custom_preset_dir_5 = custom_preset_dir_4 / "subdir" mkdir(custom_preset_dir_1) mkdir(custom_preset_dir_2) mkdir(custom_preset_dir_3) + mkdir(custom_preset_dir_4) + mkdir(custom_preset_dir_5) preset_file = custom_preset_dir_1 / "preset1.yml" with open(preset_file, "w") as f: @@ -452,10 +456,11 @@ def test_preset_include(): preset_file = custom_preset_dir_3 / "preset3.yml" with open(preset_file, "w") as f: f.write( - """ + f""" include: # uh oh - preset1 + - {custom_preset_dir_4}/preset4 config: modules: @@ -464,10 +469,38 @@ def test_preset_include(): """ ) + preset_file = custom_preset_dir_4 / "preset4.yml" + with open(preset_file, "w") as f: + f.write( + """ +include: + # uh oh + - preset5 + +config: + modules: + testpreset4: + test: zxcv +""" + ) + + preset_file = custom_preset_dir_5 / "preset5.yml" + with open(preset_file, "w") as f: + f.write( + """ +config: + modules: + testpreset5: + test: hjkl +""" + ) + preset = Preset(include=[custom_preset_dir_1 / "preset1"]) assert preset.config.modules.testpreset1.test == "asdf" assert preset.config.modules.testpreset2.test == "fdsa" assert preset.config.modules.testpreset3.test == "qwerty" + assert preset.config.modules.testpreset4.test == "zxcv" + assert preset.config.modules.testpreset5.test == "hjkl" # test custom module load directory From cf23bba798a6e62ecf2e361d7f16b2aefb38b5ad Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 21 Mar 2024 22:52:53 -0400 Subject: [PATCH 052/171] better cli tests, preset+cli tests passing --- bbot/cli.py | 49 +++++--- bbot/core/core.py | 19 ++-- bbot/core/errors.py | 4 + bbot/core/helpers/misc.py | 11 +- bbot/core/modules.py | 17 ++- bbot/modules/dockerhub.py | 2 +- bbot/modules/github_codesearch.py | 2 +- bbot/modules/github_org.py | 2 +- bbot/modules/output/emails.py | 1 - bbot/modules/output/subdomains.py | 1 - bbot/presets/cloud-enum.yml | 7 ++ bbot/presets/code-enum.yml | 4 + bbot/presets/email-enum.yml | 4 +- bbot/presets/kitchen-sink.yml | 11 ++ bbot/presets/secrets-enum.yml | 1 + bbot/presets/spider.yml | 6 +- bbot/presets/subdomain-enum.yml | 2 + bbot/presets/web-basic.yml | 4 + bbot/presets/web-thorough.yml | 7 ++ bbot/presets/web_advanced/dirbust-heavy.yml | 39 +++++++ bbot/presets/web_advanced/dirbust-light.yml | 16 +++ bbot/presets/web_advanced/paramminer.yml | 11 ++ bbot/scanner/preset/args.py | 119 ++++++++++---------- bbot/scanner/preset/path.py | 39 ++++--- bbot/scanner/preset/preset.py | 91 ++++++++++----- bbot/test/bbot_fixtures.py | 11 -- bbot/test/conftest.py | 10 ++ bbot/test/test.conf | 101 ++++++++--------- bbot/test/test_step_1/test_cli.py | 119 +++++++++++++++----- bbot/test/test_step_1/test_presets.py | 5 +- 30 files changed, 480 insertions(+), 235 deletions(-) create mode 100644 bbot/presets/cloud-enum.yml create mode 100644 bbot/presets/code-enum.yml create mode 100644 bbot/presets/kitchen-sink.yml create mode 100644 bbot/presets/secrets-enum.yml create mode 100644 bbot/presets/web-basic.yml create mode 100644 bbot/presets/web-thorough.yml create mode 100644 bbot/presets/web_advanced/dirbust-heavy.yml create mode 100644 bbot/presets/web_advanced/dirbust-light.yml create mode 100644 bbot/presets/web_advanced/paramminer.yml diff --git a/bbot/cli.py b/bbot/cli.py index 08f77f504..8203e45c3 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -12,6 +12,7 @@ from bbot.core import CORE from bbot import __version__ +from bbot.core.errors import * from bbot.core.helpers.logger import log_to_stderr log = logging.getLogger("bbot.cli") @@ -29,9 +30,14 @@ async def _main(): try: # start by creating a default scan preset - preset = Preset() + preset = Preset(_log=True, name="bbot_cli_main") # parse command line arguments and merge into preset - preset.parse_args() + try: + preset.parse_args() + except BBOTArgumentError as e: + log_to_stderr(str(e), level="WARNING") + log.trace(traceback.format_exc()) + return # ensure arguments (-c config options etc.) are valid options = preset.args.parsed @@ -51,48 +57,57 @@ async def _main(): # --version if options.version: - log.stdout(__version__) + print(__version__) sys.exit(0) return # --current-preset if options.current_preset: - log.stdout(preset.to_yaml()) + print(preset.to_yaml()) sys.exit(0) return # --current-preset-full if options.current_preset_full: - log.stdout(preset.to_yaml(full_config=True)) + print(preset.to_yaml(full_config=True)) sys.exit(0) return + # --list-presets + if options.list_presets: + print("") + print("### PRESETS ###") + print("") + for row in preset.presets_table().splitlines(): + print(row) + return + # --list-modules if options.list_modules: - log.stdout("") - log.stdout("### MODULES ###") - log.stdout("") + print("") + print("### MODULES ###") + print("") for row in preset.module_loader.modules_table(modules_to_list).splitlines(): - log.stdout(row) + print(row) return # --list-module-options if options.list_module_options: - log.stdout("") - log.stdout("### MODULE OPTIONS ###") - log.stdout("") + print("") + print("### MODULE OPTIONS ###") + print("") for row in preset.module_loader.modules_options_table(modules=modules_to_list).splitlines(): - log.stdout(row) + print(row) return # --list-flags if options.list_flags: flags = preset.flags if preset.flags else None - log.stdout("") - log.stdout("### FLAGS ###") - log.stdout("") + print("") + print("### FLAGS ###") + print("") for row in preset.module_loader.flags_table(flags=flags).splitlines(): - log.stdout(row) + print(row) return deadly_modules = [m for m in preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", [])] diff --git a/bbot/core/core.py b/bbot/core/core.py index 17a2e92e8..25dabeece 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,7 +1,10 @@ +import logging from copy import copy from pathlib import Path from omegaconf import OmegaConf +DEFAULT_CONFIG = None + class BBOTCore: @@ -12,7 +15,6 @@ def __init__(self): self.bbot_sudo_pass = None self._config = None - self._default_config = None self._custom_config = None # ensure bbot home dir @@ -21,6 +23,7 @@ def __init__(self): # bare minimum == logging self.logger + self.log = logging.getLogger("bbot.core") # PRESET TODO: add back in bbot/core/configurator/__init__.py # - check_cli_args @@ -65,17 +68,20 @@ def config(self): @property def default_config(self): - if self._default_config is None: - self._default_config = self.files_config.get_default_config() + global DEFAULT_CONFIG + if DEFAULT_CONFIG is None: + DEFAULT_CONFIG = self.files_config.get_default_config() # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(self._default_config, True) - return self._default_config + OmegaConf.set_readonly(DEFAULT_CONFIG, True) + return DEFAULT_CONFIG @default_config.setter def default_config(self, value): # we temporarily clear out the config so it can be refreshed if/when default_config changes + global DEFAULT_CONFIG self._config = None - self._default_config = value + DEFAULT_CONFIG = value + OmegaConf.set_readonly(DEFAULT_CONFIG, True) @property def custom_config(self): @@ -99,7 +105,6 @@ def merge_default(self, config): def copy(self): core_copy = copy(self) - core_copy._default_config = self._default_config.copy() core_copy._custom_config = self._custom_config.copy() return core_copy diff --git a/bbot/core/errors.py b/bbot/core/errors.py index 6f13c9875..7e463033f 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -60,3 +60,7 @@ class EnableModuleError(BBOTError): class EnableFlagError(BBOTError): pass + + +class BBOTArgumentError(BBOTError): + pass diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index f102c47eb..14cc6ea12 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -15,7 +15,6 @@ from .url import * # noqa F401 from .. import errors -from .logger import log_to_stderr from . import regexes as bbot_regexes from .names_generator import random_name, names, adjectives # noqa F401 @@ -1101,8 +1100,8 @@ def closest_match(s, choices, n=1, cutoff=0.0): return matches -def match_and_exit(s, choices, msg=None, loglevel="HUGEWARNING", exitcode=2): - """Finds the closest match from a list of choices for a given string, logs a warning, and exits the program. +def get_closest_match(s, choices, msg=None): + """Finds the closest match from a list of choices for a given string. This function is particularly useful for CLI applications where you want to validate flags or modules. @@ -1114,17 +1113,15 @@ def match_and_exit(s, choices, msg=None, loglevel="HUGEWARNING", exitcode=2): exitcode (int, optional): The exit code to use when exiting the program. Defaults to 2. Examples: - >>> match_and_exit("some_module", ["some_mod", "some_other_mod"], msg="module") + >>> get_closest_match("some_module", ["some_mod", "some_other_mod"], msg="module") # Output: Could not find module "some_module". Did you mean "some_mod"? - # Exits with code 2 """ if msg is None: msg = "" else: msg += " " closest = closest_match(s, choices) - log_to_stderr(f'Could not find {msg}"{s}". Did you mean "{closest}"?', level="HUGEWARNING") - sys.exit(2) + return f'Could not find {msg}"{s}". Did you mean "{closest}"?' def kill_children(parent_pid=None, sig=None): diff --git a/bbot/core/modules.py b/bbot/core/modules.py index fe284fedb..c6d050a57 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -5,6 +5,7 @@ import pickle import logging import importlib +import omegaconf import traceback from copy import copy from pathlib import Path @@ -112,13 +113,17 @@ def preload(self, module_dirs=None): ... } """ + new_modules = False if module_dirs is None: module_dirs = self.module_dirs for module_dir in module_dirs: if module_dir in self._module_dirs_preloaded: + log.debug(f"Already preloaded modules from {module_dir}") continue + log.debug(f"Preloading modules from {module_dir}") + new_modules = True for module_file in list_files(module_dir, filter=self.file_filter): module_name = module_file.stem module_file = module_file.resolve() @@ -130,8 +135,10 @@ def preload(self, module_dirs=None): self._all_flags.update(set(flags)) cache_key = preloaded_module.get("cache_key", ()) if module_cache_key == cache_key: + log.debug(f"Preloading {module_name} from cache") preloaded = self.preload_cache[module_name] else: + log.debug(f"Preloading {module_name} from disk") if module_dir.name == "modules": namespace = f"bbot.modules" else: @@ -167,7 +174,15 @@ def preload(self, module_dirs=None): self._module_dirs_preloaded.add(module_dir) - return self.__preloaded + # update default config with module defaults + module_config = omegaconf.OmegaConf.create( + { + "modules": self.configs(), + } + ) + self.core.merge_default(module_config) + + return new_modules @property def preload_cache(self): diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py index c06998178..de39c076f 100644 --- a/bbot/modules/dockerhub.py +++ b/bbot/modules/dockerhub.py @@ -4,7 +4,7 @@ class dockerhub(BaseModule): watched_events = ["SOCIAL", "ORG_STUB"] produced_events = ["SOCIAL", "CODE_REPOSITORY", "URL_UNVERIFIED"] - flags = ["active", "safe"] + flags = ["active", "safe", "code-enum"] meta = {"description": "Search for docker repositories of discovered orgs/usernames"} site_url = "https://hub.docker.com" diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index fdc58695b..587e10ff3 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -4,7 +4,7 @@ class github_codesearch(github): watched_events = ["DNS_NAME"] produced_events = ["CODE_REPOSITORY", "URL_UNVERIFIED"] - flags = ["passive", "subdomain-enum", "safe"] + flags = ["passive", "subdomain-enum", "safe", "code-enume"] meta = {"description": "Query Github's API for code containing the target domain name", "auth_required": True} options = {"api_key": "", "limit": 100} options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"} diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 70cba4560..95d02549e 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -4,7 +4,7 @@ class github_org(github): watched_events = ["ORG_STUB", "SOCIAL"] produced_events = ["CODE_REPOSITORY"] - flags = ["passive", "subdomain-enum", "safe"] + flags = ["passive", "subdomain-enum", "safe", "code-enum"] meta = {"description": "Query Github's API for organization and member repositories"} options = {"api_key": "", "include_members": True, "include_member_repos": False} options_desc = { diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py index e96c5d97c..1798f0135 100644 --- a/bbot/modules/output/emails.py +++ b/bbot/modules/output/emails.py @@ -4,7 +4,6 @@ class Emails(Human): watched_events = ["EMAIL_ADDRESS"] - flags = ["email-enum"] meta = {"description": "Output any email addresses found belonging to the target domain"} options = {"output_file": ""} options_desc = {"output_file": "Output to file"} diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py index bfb7174ac..b0ec08aeb 100644 --- a/bbot/modules/output/subdomains.py +++ b/bbot/modules/output/subdomains.py @@ -4,7 +4,6 @@ class Subdomains(Human): watched_events = ["DNS_NAME", "DNS_NAME_UNRESOLVED"] - flags = ["subdomain-enum"] meta = {"description": "Output only resolved, in-scope subdomains"} options = {"output_file": "", "include_unresolved": False} options_desc = {"output_file": "Output to file", "include_unresolved": "Include unresolved subdomains in output"} diff --git a/bbot/presets/cloud-enum.yml b/bbot/presets/cloud-enum.yml new file mode 100644 index 000000000..6f19c5c35 --- /dev/null +++ b/bbot/presets/cloud-enum.yml @@ -0,0 +1,7 @@ +description: Enumerate cloud resources such as storage buckets, etc. + +include: + - subdomain-enum + +flags: + - cloud-enum diff --git a/bbot/presets/code-enum.yml b/bbot/presets/code-enum.yml new file mode 100644 index 000000000..8e91e5674 --- /dev/null +++ b/bbot/presets/code-enum.yml @@ -0,0 +1,4 @@ +description: Enumerate Git repositories, Docker images, etc. + +flags: + - code-enum diff --git a/bbot/presets/email-enum.yml b/bbot/presets/email-enum.yml index 5eba4f081..9f391e28b 100644 --- a/bbot/presets/email-enum.yml +++ b/bbot/presets/email-enum.yml @@ -1,6 +1,4 @@ -# include web spider (emails are automatically extracted from HTTP_RESPONSEs) -include: - - spider +description: Enumerate email addresses from APIs, web crawling, etc. flags: - email-enum diff --git a/bbot/presets/kitchen-sink.yml b/bbot/presets/kitchen-sink.yml new file mode 100644 index 000000000..afe1ee92c --- /dev/null +++ b/bbot/presets/kitchen-sink.yml @@ -0,0 +1,11 @@ +description: Everything everywhere all at once + +include: + - subdomain-enum + - cloud-enum + - code-enum + - email-enum + - spider + - web-basic + - paramminer + - dirbust-light diff --git a/bbot/presets/secrets-enum.yml b/bbot/presets/secrets-enum.yml new file mode 100644 index 000000000..b97271f79 --- /dev/null +++ b/bbot/presets/secrets-enum.yml @@ -0,0 +1 @@ +description: diff --git a/bbot/presets/spider.yml b/bbot/presets/spider.yml index d868b76e3..3a1ef5de4 100644 --- a/bbot/presets/spider.yml +++ b/bbot/presets/spider.yml @@ -1,11 +1,13 @@ +description: Recursive web spider + modules: - httpx config: # how many links to follow in a row web_spider_distance: 2 - # don't follow links whose directory depth is higher than 3 - web_spider_depth: 3 + # don't follow links whose directory depth is higher than 4 + web_spider_depth: 4 # maximum number of links to follow per page web_spider_links_per_page: 25 diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index 8310ed158..1741a06e5 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -1,3 +1,5 @@ +description: Enumerate subdomains via APIs, brute-force + flags: - subdomain-enum diff --git a/bbot/presets/web-basic.yml b/bbot/presets/web-basic.yml new file mode 100644 index 000000000..433cb1e3c --- /dev/null +++ b/bbot/presets/web-basic.yml @@ -0,0 +1,4 @@ +description: Quick web scan + +flags: + - web-basic diff --git a/bbot/presets/web-thorough.yml b/bbot/presets/web-thorough.yml new file mode 100644 index 000000000..6b15c5794 --- /dev/null +++ b/bbot/presets/web-thorough.yml @@ -0,0 +1,7 @@ +description: Aggressive web scan + +include: + - web-basic + +flags: + - web-thorough diff --git a/bbot/presets/web_advanced/dirbust-heavy.yml b/bbot/presets/web_advanced/dirbust-heavy.yml new file mode 100644 index 000000000..effba2554 --- /dev/null +++ b/bbot/presets/web_advanced/dirbust-heavy.yml @@ -0,0 +1,39 @@ +description: Recursive web directory brute-force (aggressive) + +include: + - spider + +flags: + - iis-shortnames + +modules: + - ffuf + - wayback + +config: + modules: + iis_shortnames: + # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames + detect_only: False + ffuf: + depth: 3 + lines: 5000 + extensions: + - php + - asp + - aspx + - ashx + - asmx + - jsp + - jspx + - cfm + - zip + - conf + - config + - xml + - json + - yml + - yaml + # emit URLs from wayback + wayback: + urls: True diff --git a/bbot/presets/web_advanced/dirbust-light.yml b/bbot/presets/web_advanced/dirbust-light.yml new file mode 100644 index 000000000..be6d3fc0d --- /dev/null +++ b/bbot/presets/web_advanced/dirbust-light.yml @@ -0,0 +1,16 @@ +description: Basic web directory brute-force (surface-level directories only) + +flags: + - iis-shortnames + +modules: + - ffuf + +config: + modules: + iis_shortnames: + # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames + detect_only: False + ffuf: + # wordlist size = 1000 + lines: 1000 diff --git a/bbot/presets/web_advanced/paramminer.yml b/bbot/presets/web_advanced/paramminer.yml new file mode 100644 index 000000000..0b2c6e811 --- /dev/null +++ b/bbot/presets/web_advanced/paramminer.yml @@ -0,0 +1,11 @@ +description: Discover new web parameters via brute-force + +flags: + - web-paramminer + +modules: + - httpx + +config: + web_spider_distance: 1 + web_spider_depth: 4 diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index cd0f3d292..e88d38a68 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -1,12 +1,10 @@ import re -import sys import logging import argparse -import traceback from omegaconf import OmegaConf -from bbot.core.errors import PresetNotFoundError -from bbot.core.helpers.misc import chain_lists, match_and_exit +from bbot.core.errors import * +from bbot.core.helpers.misc import chain_lists, get_closest_match log = logging.getLogger("bbot.presets.args") @@ -55,6 +53,11 @@ class BBOTArgs: "", "bbot -l", ), + ( + "List presets", + "", + "bbot -lp", + ), ( "List flags", "", @@ -90,15 +93,17 @@ def parsed(self): return self._parsed def preset_from_args(self): - self.validate() + # the order here is important + # first we make the preset args_preset = self.preset.__class__( *self.parsed.targets, whitelist=self.parsed.whitelist, blacklist=self.parsed.blacklist, strict_scope=self.parsed.strict_scope, + name="args_preset", ) - # verbosity levels + # then we set verbosity levels (so if the user enables -d they can see debug output) if self.parsed.silent: args_preset.silent = True if self.parsed.verbose: @@ -106,42 +111,30 @@ def preset_from_args(self): if self.parsed.debug: args_preset.debug = True - # modules && flags (excluded then required then all others) + # then we load requested preset + # this is important so we can load custom module directories, pull in custom flags, module config options, etc. + for preset_arg in self.parsed.preset: + try: + args_preset.include_preset(preset_arg) + except BBOTArgumentError: + raise + except Exception as e: + raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}') + + # then we validate the modules/flags/config options + self.validate() + + # load modules & flags (excluded then required then all others) args_preset.exclude_modules = self.parsed.exclude_modules args_preset.exclude_flags = self.parsed.exclude_flags args_preset.require_flags = self.parsed.require_flags - args_preset.modules = self.parsed.modules - + for scan_module in self.parsed.modules: + args_preset.add_module(scan_module, module_type="scan") for output_module in self.parsed.output_modules: args_preset.add_module(output_module, module_type="output") args_preset.flags = self.parsed.flags - # additional custom presets / config options - preset_args = [] - for preset_param in self.parsed.preset: - try: - # first try to load as a file - custom_preset = self.preset.from_yaml_file(preset_param) - args_preset.merge(custom_preset) - except PresetNotFoundError as e: - log.debug(e) - try: - # if that fails, try to parse as key=value syntax - cli_config = OmegaConf.from_cli([preset_param]) - preset_args.append(preset_param) - args_preset.core.merge_custom(cli_config) - except Exception as e: - log.error(f"Error parsing command-line config: {e}") - log.trace(traceback.format_exc()) - sys.exit(2) - except Exception as e: - log.error(f"Error parsing custom config at {preset_param}: {e}") - log.trace(traceback.format_exc()) - sys.exit(2) - - self.parsed.preset = preset_args - # dependencies if self.parsed.retry_deps: args_preset.core.custom_config["deps_behavior"] = "retry_failed" @@ -156,6 +149,14 @@ def preset_from_args(self): args_preset.scan_name = self.parsed.name args_preset.output_dir = self.parsed.output_dir + # CLI config options (dot-syntax) + for config_arg in self.parsed.config: + try: + # if that fails, try to parse as key=value syntax + args_preset.core.merge_custom(OmegaConf.from_cli([config_arg])) + except Exception as e: + raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}') + return args_preset def create_parser(self, *args, **kwargs): @@ -182,6 +183,24 @@ def create_parser(self, *args, **kwargs): action="store_true", help="Don't consider subdomains of target/whitelist to be in-scope", ) + presets = p.add_argument_group(title="Presets") + presets.add_argument( + "-p", + "--preset", + nargs="*", + help="Enable BBOT preset(s)", + metavar="PRESET", + default=[], + ) + presets.add_argument( + "-c", + "--config", + nargs="*", + help="Custom config options in key=value format: e.g. 'modules.shodan.api_key=1234'", + metavar="CONFIG", + default=[], + ) + presets.add_argument("-lp", "--list-presets", action="store_true", help=f"List available presets.") modules = p.add_argument_group(title="Modules") modules.add_argument( "-m", @@ -239,16 +258,6 @@ def create_parser(self, *args, **kwargs): "--output-dir", metavar="DIR", ) - scan.add_argument( - "-p", - "--preset", - "-c", - "--config", - nargs="*", - help="Custom preset file(s), or config options in key=value format: 'modules.shodan.api_key=1234'", - metavar="CONFIG", - default=[], - ) scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") @@ -304,8 +313,10 @@ def sanitize_args(self): def validate(self): # validate config options sentinel = object() - all_options = None - for c in self.parsed.preset: + all_options = set(self.preset.core.default_config.keys()) - {"modules"} + for module_options in self.preset.module_loader.modules_options().values(): + all_options.update(set(o[0] for o in module_options)) + for c in self.parsed.config: c = c.split("=")[0].strip() v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel) # if option isn't in the default config @@ -313,27 +324,21 @@ def validate(self): # skip if it's excluded from validation if self.exclude_from_validation.match(c): continue - if all_options is None: - modules_options = set() - for module_options in self.preset.module_loader.modules_options().values(): - modules_options.update(set(o[0] for o in module_options)) - global_options = set(self.preset.core.default_config.keys()) - {"modules"} - all_options = global_options.union(modules_options) # otherwise, ensure it exists as a module option - match_and_exit(c, all_options, msg="module option") + raise BBOTArgumentError(get_closest_match(c, all_options, msg="module option")) # validate modules for m in self.parsed.modules: if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") + raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="module")) for m in self.parsed.exclude_modules: if m not in self._module_choices: - match_and_exit(m, self._module_choices, msg="module") + raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="module")) for m in self.parsed.output_modules: if m not in self._output_module_choices: - match_and_exit(m, self._output_module_choices, msg="output module") + raise BBOTArgumentError(get_closest_match(m, self._output_module_choices, msg="output module")) # validate flags for f in set(self.parsed.flags + self.parsed.require_flags + self.parsed.exclude_flags): if f not in self._flag_choices: - match_and_exit(f, self._flag_choices, msg="flag") + raise BBOTArgumentError(get_closest_match(f, self._flag_choices, msg="flag")) diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index 7c89dc648..0a6b5a46b 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from bbot.core.errors import PresetNotFoundError +from bbot.core.errors import * log = logging.getLogger("bbot.presets.path") @@ -15,30 +15,32 @@ class PresetPath: def __init__(self): self.paths = [DEFAULT_PRESET_PATH] - self.add_path(Path.cwd()) def find(self, filename): - filename = Path(filename).resolve() - self.add_path(filename.parent) - if filename.is_file(): - log.hugesuccess(filename) - return filename - extension = filename.suffix.lower() + filename_path = Path(filename).resolve() + extension = filename_path.suffix.lower() file_candidates = set() - if not extension: - file_candidates.add(filename.stem) - for ext in (".yaml", ".yml"): - if extension != ext: - file_candidates.add(f"{filename.stem}{ext}") + extension_candidates = {".yaml", ".yml"} + if extension: + extension_candidates.add(extension.lower()) + else: + file_candidates.add(filename_path.stem) + for ext in extension_candidates: + file_candidates.add(f"{filename_path.stem}{ext}") file_candidates = sorted(file_candidates) file_candidates_str = ",".join([str(s) for s in file_candidates]) - log.debug(f"Searching for preset in {self}, file candidates: {file_candidates_str}") - for path in self.paths: + paths_to_search = self.paths + if "/" in str(filename): + if filename_path.parent not in paths_to_search: + paths_to_search.append(filename_path.parent) + log.debug(f"Searching for preset in {paths_to_search}, file candidates: {file_candidates_str}") + for path in paths_to_search: for candidate in file_candidates: for file in path.rglob(candidate): log.verbose(f'Found preset matching "{filename}" at {file}') + self.add_path(file.parent) return file.resolve() - raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') + raise BBOTError(f'Could not find preset at "{filename}" - file does not exist') def __str__(self): return ":".join([str(s) for s in self.paths]) @@ -47,10 +49,15 @@ def add_path(self, path): path = Path(path).resolve() if path in self.paths: return + if any(path.is_relative_to(p) for p in self.paths): + return if not path.is_dir(): log.debug(f'Path "{path.resolve()}" is not a directory') return self.paths.append(path) + def __iter__(self): + yield from self.paths + PRESET_PATH = PresetPath() diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index d4f06dc9b..82f20422e 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -2,6 +2,7 @@ import yaml import logging import omegaconf +import traceback from copy import copy from pathlib import Path from contextlib import suppress @@ -9,8 +10,9 @@ from .path import PRESET_PATH from bbot.core import CORE +from bbot.core.errors import * from bbot.core.event.base import make_event -from bbot.core.errors import EnableFlagError, EnableModuleError, PresetNotFoundError, ValidationError +from bbot.core.helpers.misc import make_table log = logging.getLogger("bbot.presets") @@ -39,8 +41,12 @@ def __init__( include=None, output_dir=None, scan_name=None, + name=None, + description=None, _exclude=None, + _log=False, ): + self._log = _log self.scan = None self._args = None self._environ = None @@ -60,6 +66,8 @@ def __init__( self.output_dir = output_dir self.scan_name = scan_name + self.name = name or "" + self.description = description or "" self._preset_files_loaded = set() if _exclude is not None: @@ -132,9 +140,6 @@ def merge(self, other): self.core.merge_custom(other.core.custom_config) self.module_loader.core = self.core # module dirs - if other.module_dirs: - self.refresh_module_loader() - # TODO: find-and-replace module configs # modules + flags # establish requirements / exclusions first self.exclude_modules = set(self.exclude_modules).union(set(other.exclude_modules)) @@ -254,7 +259,7 @@ def _modules_setter(self, modules, module_type="scan"): def add_module(self, module_name, module_type="scan"): # log.info(f'Adding "{module_name}": {module_type}') if module_name in self.exclude_modules: - log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') return try: preloaded = self.module_loader.preloaded()[module_name] @@ -265,28 +270,28 @@ def add_module(self, module_name, module_type="scan"): _module_type = preloaded.get("type", "scan") if module_type: if _module_type != module_type: - log.verbose( - f'Not enabling module "{module_name}" because its type ({_module_type}) is not "{module_type}"' + self.log_verbose( + f'Not adding module "{module_name}" because its type ({_module_type}) is not "{module_type}"' ) return if _module_type == "scan": for f in module_flags: if f in self.exclude_flags: - log.verbose(f'Skipping module "{module_name}" because it\'s excluded') + self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') return if self.require_flags and not any(f in self.require_flags for f in module_flags): - log.verbose( + self.log_verbose( f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' ) return if module_name not in self.modules: - log.verbose(f'Enabling module "{module_name}"') + self.log_verbose(f'Adding module "{module_name}"') self.modules.add(module_name) for module_dep in preloaded.get("deps", {}).get("modules", []): if module_dep not in self.modules: - log.verbose(f'Enabling module "{module_dep}" because {module_name} depends on it') + self.log_verbose(f'Adding module "{module_dep}" because {module_name} depends on it') self.add_module(module_dep) @property @@ -307,6 +312,7 @@ def flags(self): @flags.setter def flags(self, flags): + log.debug(f"{self.name}: setting flags to {flags}") if isinstance(flags, str): flags = [flags] for flag in flags: @@ -348,7 +354,7 @@ def add_required_flag(self, flag): for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) if flag not in module_flags: - log.verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') + self.log_verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') self.modules.remove(module) def add_excluded_flag(self, flag): @@ -356,14 +362,14 @@ def add_excluded_flag(self, flag): for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) if flag in module_flags: - log.verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') + self.log_verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') self.modules.remove(module) def add_excluded_module(self, module): self.exclude_modules.add(module) for module in list(self.scan_modules): if module in self.exclude_modules: - log.verbose(f'Removing module "{module}" because it\'s excluded') + self.log_verbose(f'Removing module "{module}" because it\'s excluded') self.modules.remove(module) def preloaded_module(self, module): @@ -439,20 +445,9 @@ def module_loader(self): from bbot.core.modules import module_loader self._module_loader = module_loader - self.refresh_module_loader() return self._module_loader - def refresh_module_loader(self): - self.module_loader.preload() - # update default config with module defaults - module_config = omegaconf.OmegaConf.create( - { - "modules": self.module_loader.configs(), - } - ) - self.core.merge_default(module_config) - @property def environ(self): if self._environ is None: @@ -500,7 +495,7 @@ def whitelisted(self, e): return e in self.whitelist @classmethod - def from_dict(cls, preset_dict, _exclude=None): + def from_dict(cls, preset_dict, name=None, _exclude=None): new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -520,12 +515,14 @@ def from_dict(cls, preset_dict, _exclude=None): include=preset_dict.get("include", []), scan_name=preset_dict.get("scan_name"), output_dir=preset_dict.get("output_dir"), + name=preset_dict.get("name", name), + description=preset_dict.get("description"), _exclude=_exclude, ) return new_preset def include_preset(self, filename): - log.debug(f'Including preset "{filename}"') + self.log_debug(f'Including preset "{filename}"') preset_filename = PRESET_PATH.find(filename) preset_from_yaml = self.from_yaml_file(preset_filename, _exclude=self._preset_files_loaded) if preset_from_yaml is not False: @@ -543,13 +540,13 @@ def from_yaml_file(cls, filename, _exclude=None): _exclude = set() filename = Path(filename).resolve() if _exclude is not None and filename in _exclude: - log.debug(f"Not merging {filename} because it was already loaded {_exclude}") + log.debug(f"Not loading {filename} because it was already loaded {_exclude}") return False - log.debug(f"Merging {filename} because it's not in {_exclude}") + log.debug(f"Loading {filename} because it's not in excluded list ({_exclude})") _exclude = set(_exclude) _exclude.add(filename) try: - return cls.from_dict(omegaconf.OmegaConf.load(filename), _exclude=_exclude) + return cls.from_dict(omegaconf.OmegaConf.load(filename), name=filename.stem, _exclude=_exclude) except FileNotFoundError: raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') @@ -618,3 +615,37 @@ def to_dict(self, include_target=False, full_config=False): def to_yaml(self, include_target=False, full_config=False, sort_keys=False): preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) + + @classmethod + def all_presets(cls): + preset_files = dict() + for ext in ("yml", "yaml"): + for preset_path in PRESET_PATH: + for yaml_file in preset_path.rglob(f"**/*.{ext}"): + try: + loaded_preset = cls.from_yaml_file(yaml_file) + category = str(yaml_file.relative_to(preset_path).parent) + if category == ".": + category = "default" + preset_files[yaml_file] = (loaded_preset, category) + except Exception as e: + log.warning(f'Failed to load preset at "{yaml_file}": {e}') + log.trace(traceback.format_exc()) + continue + return preset_files + + def presets_table(self): + table = [] + header = ["Preset", "Category", "Description", "Modules"] + for loaded_preset, category in self.all_presets().values(): + modules = ", ".join(sorted(loaded_preset.scan_modules)) + table.append([loaded_preset.name, category, loaded_preset.description, modules]) + return make_table(table, header) + + def log_verbose(self, msg): + if self._log: + log.verbose(f"preset {self.name}: {msg}") + + def log_debug(self, msg): + if self._log: + self.log_debug(f"preset {self.name}: {msg}") diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index a451cdd45..46a5b0045 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -1,4 +1,3 @@ -import os import dns import sys import pytest @@ -8,11 +7,9 @@ import tldextract import pytest_httpserver from pathlib import Path -from omegaconf import OmegaConf from werkzeug.wrappers import Request -from bbot.core import CORE from bbot.core.helpers.misc import mkdir @@ -29,14 +26,6 @@ def match_data(self, request: Request) -> bool: pytest_httpserver.httpserver.RequestMatcher = SubstringRequestMatcher - -test_config = OmegaConf.load(Path(__file__).parent / "test.conf") -if test_config.get("debug", False): - os.environ["BBOT_DEBUG"] = "True" - -CORE.merge_custom(test_config) - -from .bbot_fixtures import * # noqa: F401 from bbot.core.errors import * # noqa: F401 # silence pytest_httpserver diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index b60a0633d..3b5775b2e 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,15 +1,25 @@ +import os import ssl import shutil import pytest import asyncio import logging from pathlib import Path +from omegaconf import OmegaConf from pytest_httpserver import HTTPServer +from bbot.core import CORE from bbot.core.helpers.misc import execute_sync_or_async from bbot.core.helpers.interactsh import server_list as interactsh_servers +test_config = OmegaConf.load(Path(__file__).parent / "test.conf") +if test_config.get("debug", False): + os.environ["BBOT_DEBUG"] = "True" + +CORE.merge_default(test_config) + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_sessionfinish(session, exitstatus): # Remove handlers from all loggers to prevent logging errors at exit diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 1c6e18750..ba8367461 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -1,52 +1,51 @@ home: /tmp/.bbot_test -config: - modules: - massdns: - wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt - ffuf: - prefix_busting: true - ipneighbor: - test_option: ipneighbor - output_modules: - http: - url: http://127.0.0.1:11111 - username: username - password: password - bearer: bearer - websocket: - url: ws://127.0.0.1/ws:11111 - token: asdf - neo4j: - uri: bolt://127.0.0.1:11111 - python: - test_option: asdf - internal_modules: - speculate: - test_option: speculate - http_proxy: - http_headers: { "test": "header" } - ssl_verify: false - scope_search_distance: 0 - scope_report_distance: 0 - scope_dns_search_distance: 1 - plumbus: asdf - dns_debug: false - user_agent: "BBOT Test User-Agent" - http_debug: false - agent_url: ws://127.0.0.1:8765 - agent_token: test - dns_resolution: false - dns_timeout: 1 - speculate: false - excavate: false - aggregate: false - omit_event_types: [] - debug: true - dns_wildcard_ignore: - - blacklanternsecurity.com - - fakedomain - - notreal - - google - - google.com - - example.com - - evilcorp.com +modules: + massdns: + wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt + ffuf: + prefix_busting: true + ipneighbor: + test_option: ipneighbor +output_modules: + http: + url: http://127.0.0.1:11111 + username: username + password: password + bearer: bearer + websocket: + url: ws://127.0.0.1/ws:11111 + token: asdf + neo4j: + uri: bolt://127.0.0.1:11111 + python: + test_option: asdf +internal_modules: + speculate: + test_option: speculate +http_proxy: +http_headers: { "test": "header" } +ssl_verify: false +scope_search_distance: 0 +scope_report_distance: 0 +scope_dns_search_distance: 1 +plumbus: asdf +dns_debug: false +user_agent: "BBOT Test User-Agent" +http_debug: false +agent_url: ws://127.0.0.1:8765 +agent_token: test +dns_resolution: false +dns_timeout: 1 +speculate: false +excavate: false +aggregate: false +omit_event_types: [] +debug: true +dns_wildcard_ignore: + - blacklanternsecurity.com + - fakedomain + - notreal + - google + - google.com + - example.com + - evilcorp.com diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index cc40b722e..e70cdfe56 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -2,14 +2,13 @@ @pytest.mark.asyncio -async def test_cli_args(monkeypatch, bbot_config): +async def test_cli_args(monkeypatch, capsys): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - home_dir = Path(bbot_config["home"]) - scans_home = home_dir / "scans" + scans_home = bbot_test_dir / "scans" # basic scan monkeypatch.setattr( @@ -41,21 +40,6 @@ async def test_cli_args(monkeypatch, bbot_config): dns_success = True assert ip_success and dns_success, "IP_ADDRESS and/or DNS_NAME are not present in output.txt" - # nonexistent module - monkeypatch.setattr("sys.argv", ["bbot", "-m", "asdf"]) - with pytest.raises(EnableModuleError): - result = await cli._main() - - # nonexistent output module - monkeypatch.setattr("sys.argv", ["bbot", "-om", "asdf"]) - with pytest.raises(EnableModuleError): - result = await cli._main() - - # nonexistent flag - monkeypatch.setattr("sys.argv", ["bbot", "-f", "asdf"]) - with pytest.raises(EnableFlagError): - result = await cli._main() - # show version monkeypatch.setattr("sys.argv", ["bbot", "--version"]) result = await cli._main() @@ -142,7 +126,7 @@ async def test_cli_args(monkeypatch, bbot_config): # assert success, "--install-all-deps failed for at least one module" -def test_config_validation(monkeypatch, capsys, bbot_config): +def test_cli_config_validation(monkeypatch, capsys): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) @@ -163,7 +147,7 @@ def test_config_validation(monkeypatch, capsys, bbot_config): assert 'Did you mean "web_spider_distance"?' in captured.err -def test_module_validation(monkeypatch, capsys, bbot_config): +def test_cli_module_validation(monkeypatch, capsys): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) @@ -171,8 +155,7 @@ def test_module_validation(monkeypatch, capsys, bbot_config): # incorrect module monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdnss"]) - with pytest.raises(EnableModuleError): - cli.main() + cli.main() captured = capsys.readouterr() assert 'Could not find module "massdnss"' in captured.err assert 'Did you mean "massdns"?' in captured.err @@ -186,16 +169,100 @@ def test_module_validation(monkeypatch, capsys, bbot_config): # incorrect output module monkeypatch.setattr("sys.argv", ["bbot", "-om", "neoo4j"]) - with pytest.raises(EnableModuleError): - cli.main() + cli.main() captured = capsys.readouterr() assert 'Could not find output module "neoo4j"' in captured.err assert 'Did you mean "neo4j"?' in captured.err # incorrect flag monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomainenum"]) - with pytest.raises(EnableFlagError): - cli.main() + cli.main() + captured = capsys.readouterr() + assert 'Could not find flag "subdomainenum"' in captured.err + assert 'Did you mean "subdomain-enum"?' in captured.err + + # incorrect excluded flag + monkeypatch.setattr("sys.argv", ["bbot", "-ef", "subdomainenum"]) + cli.main() + captured = capsys.readouterr() + assert 'Could not find flag "subdomainenum"' in captured.err + assert 'Did you mean "subdomain-enum"?' in captured.err + + # incorrect required flag + monkeypatch.setattr("sys.argv", ["bbot", "-rf", "subdomainenum"]) + cli.main() captured = capsys.readouterr() assert 'Could not find flag "subdomainenum"' in captured.err assert 'Did you mean "subdomain-enum"?' in captured.err + + +def test_cli_presets(monkeypatch, capsys): + import yaml + from bbot import cli + + monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) + monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + + preset_dir = bbot_test_dir / "test_cli_presets" + preset_dir.mkdir(exist_ok=True) + + preset1_file = preset_dir / "preset1.conf" + with open(preset1_file, "w") as f: + f.write( + """ +config: + http_proxy: http://proxy1 + """ + ) + + preset2_file = preset_dir / "preset2.yml" + with open(preset2_file, "w") as f: + f.write( + """ +config: + http_proxy: http://proxy2 + """ + ) + + # test reading single preset + monkeypatch.setattr("sys.argv", ["bbot", "-p", str(preset1_file.resolve()), "--current-preset"]) + cli.main() + captured = capsys.readouterr() + stdout_preset = yaml.safe_load(captured.out) + assert stdout_preset["config"]["http_proxy"] == "http://proxy1" + + # preset overrides preset + monkeypatch.setattr( + "sys.argv", ["bbot", "-p", str(preset2_file.resolve()), str(preset1_file.resolve()), "--current-preset"] + ) + cli.main() + captured = capsys.readouterr() + stdout_preset = yaml.safe_load(captured.out) + assert stdout_preset["config"]["http_proxy"] == "http://proxy1" + + # override other way + monkeypatch.setattr( + "sys.argv", ["bbot", "-p", str(preset1_file.resolve()), str(preset2_file.resolve()), "--current-preset"] + ) + cli.main() + captured = capsys.readouterr() + stdout_preset = yaml.safe_load(captured.out) + assert stdout_preset["config"]["http_proxy"] == "http://proxy2" + + # cli config overrides all presets + monkeypatch.setattr( + "sys.argv", + [ + "bbot", + "-p", + str(preset1_file.resolve()), + str(preset2_file.resolve()), + "-c", + "http_proxy=asdf", + "--current-preset", + ], + ) + cli.main() + captured = capsys.readouterr() + stdout_preset = yaml.safe_load(captured.out) + assert stdout_preset["config"]["http_proxy"] == "asdf" diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index b47f63087..98dc4b9fc 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -474,7 +474,6 @@ def test_preset_include(): f.write( """ include: - # uh oh - preset5 config: @@ -495,7 +494,7 @@ def test_preset_include(): """ ) - preset = Preset(include=[custom_preset_dir_1 / "preset1"]) + preset = Preset(include=[str(custom_preset_dir_1 / "preset1")]) assert preset.config.modules.testpreset1.test == "asdf" assert preset.config.modules.testpreset2.test == "fdsa" assert preset.config.modules.testpreset3.test == "qwerty" @@ -510,3 +509,5 @@ def test_preset_include(): # what if you specify flags in one preset # but another preset (loaded later) has more custom modules that match that flag # what if you specify a flag that's only on custom modules? Will it be rejected as invalid? + +# cli test: nonexistent / invalid preset From 9549091442d92ea32dd37f776a0aae0c60b76e63 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 22 Mar 2024 16:37:57 -0400 Subject: [PATCH 053/171] preset conditions + tests --- bbot/cli.py | 30 ++++++++------- bbot/core/errors.py | 8 ++++ bbot/presets/subdomain-enum.yml | 2 + bbot/scanner/preset/args.py | 9 ++++- bbot/scanner/preset/conditions.py | 54 +++++++++++++++++++++++++++ bbot/scanner/preset/preset.py | 29 +++++++++++++- bbot/test/test_step_1/test_cli.py | 28 +++++++++++++- bbot/test/test_step_1/test_presets.py | 42 ++++++++++++++++++++- poetry.lock | 5 +-- pyproject.toml | 1 + 10 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 bbot/scanner/preset/conditions.py diff --git a/bbot/cli.py b/bbot/cli.py index 8203e45c3..6cd2beb97 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -61,18 +61,6 @@ async def _main(): sys.exit(0) return - # --current-preset - if options.current_preset: - print(preset.to_yaml()) - sys.exit(0) - return - - # --current-preset-full - if options.current_preset_full: - print(preset.to_yaml(full_config=True)) - sys.exit(0) - return - # --list-presets if options.list_presets: print("") @@ -117,7 +105,23 @@ async def _main(): log.hugewarning(f"Please specify --allow-deadly to continue") return False - scan = Scanner(preset=preset) + try: + scan = Scanner(preset=preset) + except PresetAbortError as e: + log.warning(str(e)) + return + + # --current-preset + if options.current_preset: + print(scan.preset.to_yaml()) + sys.exit(0) + return + + # --current-preset-full + if options.current_preset_full: + print(scan.preset.to_yaml(full_config=True)) + sys.exit(0) + return # --install-all-deps if options.install_all_deps: diff --git a/bbot/core/errors.py b/bbot/core/errors.py index 7e463033f..e50e581cd 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -64,3 +64,11 @@ class EnableFlagError(BBOTError): class BBOTArgumentError(BBOTError): pass + + +class PresetConditionError(BBOTError): + pass + + +class PresetAbortError(PresetConditionError): + pass diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index 1741a06e5..24f9d1dac 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -8,6 +8,8 @@ output_modules: config: modules: + c99: + myoption: "#{MYOPTION}" stdout: format: text # only output DNS_NAMEs to the console diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index e88d38a68..5c9d8202c 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -119,7 +119,7 @@ def preset_from_args(self): except BBOTArgumentError: raise except Exception as e: - raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}') + raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e} (-d to debug)') # then we validate the modules/flags/config options self.validate() @@ -148,6 +148,7 @@ def preset_from_args(self): # other scan options args_preset.scan_name = self.parsed.name args_preset.output_dir = self.parsed.output_dir + args_preset.force = self.parsed.force # CLI config options (dot-syntax) for config_arg in self.parsed.config: @@ -261,7 +262,11 @@ def create_parser(self, *args, **kwargs): scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") - scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") + scan.add_argument( + "--force", + action="store_true", + help="Run scan even in the case of condition violations or failed module setups", + ) scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") scan.add_argument( diff --git a/bbot/scanner/preset/conditions.py b/bbot/scanner/preset/conditions.py new file mode 100644 index 000000000..54978c2b0 --- /dev/null +++ b/bbot/scanner/preset/conditions.py @@ -0,0 +1,54 @@ +import logging + +from bbot.core.errors import * + +log = logging.getLogger("bbot.preset.conditions") + +JINJA_ENV = None + + +class ConditionEvaluator: + def __init__(self, preset): + self.preset = preset + + @property + def context(self): + return { + "preset": self.preset, + "config": self.preset.config, + "abort": self.abort, + "warn": self.warn, + } + + def abort(self, message): + if not self.preset.force: + raise PresetAbortError(message) + + def warn(self, message): + log.warning(message) + + def evaluate(self): + context = self.context + already_evaluated = set() + for preset_name, condition in self.preset.conditions: + condition_str = str(condition) + if condition_str not in already_evaluated: + already_evaluated.add(condition_str) + try: + self.check_condition(condition_str, context) + except PresetAbortError as e: + raise PresetAbortError(f'Preset "{preset_name}" requested abort: {e} (--force to override)') + + @property + def jinja_env(self): + from jinja2.sandbox import SandboxedEnvironment + + global JINJA_ENV + if JINJA_ENV is None: + JINJA_ENV = SandboxedEnvironment() + return JINJA_ENV + + def check_condition(self, condition_str, context): + log.debug(f'Evaluating condition "{repr(condition_str)}"') + template = self.jinja_env.from_string(condition_str) + template.render(context) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 82f20422e..f6f5c8e30 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -43,6 +43,8 @@ def __init__( scan_name=None, name=None, description=None, + conditions=None, + force=False, _exclude=None, _log=False, ): @@ -60,6 +62,8 @@ def __init__( self._exclude_flags = set() self._flags = set() + self.force = force + self._verbose = False self._debug = False self._silent = False @@ -68,6 +72,10 @@ def __init__( self.scan_name = scan_name self.name = name or "" self.description = description or "" + self.conditions = [] + if conditions is not None: + for condition in conditions: + self.conditions.append((self.name, condition)) self._preset_files_loaded = set() if _exclude is not None: @@ -108,6 +116,8 @@ def __init__( self.blacklist = Target(*blacklist) # include other presets + if include and not isinstance(include, (list, tuple, set)): + include = [include] if include: for included_preset in include: self.include_preset(included_preset) @@ -169,6 +179,11 @@ def merge(self, other): self.scan_name = other.scan_name if other.output_dir is not None: self.output_dir = other.output_dir + # conditions + if other.conditions: + self.conditions.extend(other.conditions) + # misc + self.force = self.force | other.force def bake(self): """ @@ -188,6 +203,13 @@ def bake(self): os.environ.clear() os.environ.update(os_environ) + # evaluate conditions + if baked_preset.conditions: + from .conditions import ConditionEvaluator + + evaluator = ConditionEvaluator(baked_preset) + evaluator.evaluate() + return baked_preset def parse_args(self): @@ -512,11 +534,12 @@ def from_dict(cls, preset_dict, name=None, _exclude=None): config=preset_dict.get("config"), strict_scope=preset_dict.get("strict_scope", False), module_dirs=preset_dict.get("module_dirs", []), - include=preset_dict.get("include", []), + include=list(preset_dict.get("include", [])), scan_name=preset_dict.get("scan_name"), output_dir=preset_dict.get("output_dir"), name=preset_dict.get("name", name), description=preset_dict.get("description"), + conditions=preset_dict.get("conditions", []), _exclude=_exclude, ) return new_preset @@ -610,6 +633,10 @@ def to_dict(self, include_target=False, full_config=False): if self.scan_name: preset_dict["output_dir"] = self.output_dir + # conditions + if self.conditions: + preset_dict["conditions"] = [c[-1] for c in self.conditions] + return preset_dict def to_yaml(self, include_target=False, full_config=False, sort_keys=False): diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index e70cdfe56..309699d7f 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -1,3 +1,6 @@ +import os +import sys + from ..bbot_fixtures import * @@ -44,36 +47,59 @@ async def test_cli_args(monkeypatch, capsys): monkeypatch.setattr("sys.argv", ["bbot", "--version"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert captured.out.count(".") > 1 # show current preset - monkeypatch.setattr("sys.argv", ["bbot", "--current-preset"]) + monkeypatch.setattr("sys.argv", ["bbot", "-c", "http_proxy=currentpresettest", "--current-preset"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert " http_proxy: currentpresettest" in captured.out # show current preset (full) monkeypatch.setattr("sys.argv", ["bbot", "--current-preset-full"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert " api_key: ''" in captured.out # list modules monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + # internal modules + assert "| excavate" in captured.out + # output modules + assert "| csv" in captured.out + # scan modules + assert "| wayback" in captured.out # list module options monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert "| modules.wayback.urls" in captured.out + assert "| bool" in captured.out + assert "| emit URLs in addition to DNS_NAMEs" in captured.out + assert "| False" in captured.out # list flags monkeypatch.setattr("sys.argv", ["bbot", "--list-flags"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert "| safe" in captured.out + assert "| Non-intrusive, safe to run" in captured.out # no args monkeypatch.setattr("sys.argv", ["bbot"]) result = await cli._main() assert result == None + captured = capsys.readouterr() + assert "Target:\n -t TARGET [TARGET ...]" in captured.out # enable module by flag monkeypatch.setattr("sys.argv", ["bbot", "-f", "report"]) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 98dc4b9fc..b8f9f9978 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -1,6 +1,6 @@ from ..bbot_fixtures import * # noqa F401 -from bbot.scanner import Preset +from bbot.scanner import Scanner, Preset def test_core(): @@ -502,6 +502,46 @@ def test_preset_include(): assert preset.config.modules.testpreset5.test == "hjkl" +def test_preset_conditions(): + custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir" + custom_preset_dir_2 = custom_preset_dir_1 / "preset_subdir" + mkdir(custom_preset_dir_1) + mkdir(custom_preset_dir_2) + + preset_file_1 = custom_preset_dir_1 / "preset1.yml" + with open(preset_file_1, "w") as f: + f.write( + """ +include: + - preset2 +""" + ) + + preset_file_2 = custom_preset_dir_2 / "preset2.yml" + with open(preset_file_2, "w") as f: + f.write( + """ +conditions: + - | + {% if config.web_spider_distance == 3 and config.web_spider_depth == 4 %} + {{ abort("web spider is too aggressive") }} + {% endif %} +""" + ) + + preset = Preset(include=[preset_file_1]) + assert preset.conditions + + scan = Scanner(preset=preset) + assert scan.preset.conditions + + preset2 = Preset(config={"web_spider_distance": 3, "web_spider_depth": 4}) + preset.merge(preset2) + + with pytest.raises(PresetAbortError): + Scanner(preset=preset) + + # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation # what if you specify -c modules.custommodule.option diff --git a/poetry.lock b/poetry.lock index 89d49900c..7e7f87560 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2048,7 +2048,6 @@ optional = false python-versions = "*" files = [ {file = "requests-file-2.0.0.tar.gz", hash = "sha256:20c5931629c558fda566cacc10cfe2cd502433e628f568c34c80d96a0cc95972"}, - {file = "requests_file-2.0.0-py2.py3-none-any.whl", hash = "sha256:3e493d390adb44aa102ebea827a48717336d5268968c370eaf19abaf5cae13bf"}, ] [package.dependencies] @@ -2447,4 +2446,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f3ba496b0b736084df4d09d85e37788b2722679ef9a3a5ca000139a3ed142081" +content-hash = "aa3357393e3bcf0e51bb097f2d9cfd46533e0d74201c0d4ec22aa71f58955e7e" diff --git a/pyproject.toml b/pyproject.toml index 1c1b6ee51..d889e613f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ cloudcheck = ">=2.1.0.181,<4.0.0.0" tldextract = "^5.1.1" cachetools = "^5.3.2" socksio = "^1.0.0" +jinja2 = "^3.1.3" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From 7817365b6df17ad020d4b7ca5e48760bf257daaa Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 25 Mar 2024 17:14:32 -0400 Subject: [PATCH 054/171] starting work on documentation --- README.md | 40 ++++--- bbot/modules/github_codesearch.py | 2 +- bbot/modules/gitlab.py | 2 +- bbot/scanner/preset/preset.py | 157 ++++++++++++++++--------- bbot/scripts/docs.py | 27 +++-- docs/modules/list_of_modules.md | 10 +- docs/scanning/advanced.md | 42 +++---- docs/scanning/configuration.md | 102 ++++++++-------- docs/scanning/index.md | 6 +- docs/scanning/presets.md | 188 ++++++++++++++++++++++++++++++ 10 files changed, 411 insertions(+), 165 deletions(-) create mode 100644 docs/scanning/presets.md diff --git a/README.md b/README.md index d7a51c3cc..876ddf5c2 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,7 @@ For a full list of modules, including the data types consumed and emitted by eac |------------------|-------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | safe | 79 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | | passive | 58 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 45 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | +| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | | active | 43 | Makes active connections to target systems | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dockerhub, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer | | web-thorough | 29 | More advanced web scanning functionality | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | | aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | @@ -276,14 +276,16 @@ For a full list of modules, including the data types consumed and emitted by eac | cloud-enum | 12 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth | | slow | 9 | May take a long time to complete | bucket_digitalocean, dastardly, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | | affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | -| email-enum | 7 | Enumerates email addresses | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert | +| email-enum | 6 | Enumerates email addresses | dehashed, emailformat, hunterio, pgp, skymem, sslcert | | deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | | portscan | 3 | Discovers open ports | internetdb, masscan, nmap | | web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | | baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | +| code-enum | 2 | | dockerhub, github_org | | iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | | report | 2 | Generates a report at the end of the scan | affiliates, asn | | social-enum | 2 | Enumerates social media | httpx, social | +| code-enume | 1 | | github_codesearch | | service-enum | 1 | Identifies protocols running on open ports | fingerprintx | | subdomain-hijack | 1 | Detects hijackable subdomains | baddns | | web-screenshots | 1 | Takes screenshots of web pages | gowitness | @@ -293,21 +295,21 @@ For a full list of modules, including the data types consumed and emitted by eac BBOT can save its data to TXT, CSV, JSON, and tons of other destinations including [Neo4j](https://www.blacklanternsecurity.com/bbot/scanning/output/#neo4j), [Splunk](https://www.blacklanternsecurity.com/bbot/scanning/output/#splunk), and [Discord](https://www.blacklanternsecurity.com/bbot/scanning/output/#discord-slack-teams). For instructions on how to use these, see [Output Modules](https://www.blacklanternsecurity.com/bbot/scanning/output). -| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | -|-----------------|--------|-----------------|-----------------------------------------------------------------------------------------|----------------|--------------------------------------------------------------------------------------------------|---------------------------| -| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | -| csv | output | No | Output to CSV | | * | | -| discord | output | No | Message a Discord channel when certain events are encountered | | * | | -| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | -| http | output | No | Send every event to a custom URL via a web request | | * | | -| human | output | No | Output to text | | * | | -| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | -| neo4j | output | No | Output to Neo4j | | * | | -| python | output | No | Output via Python API | | * | | -| slack | output | No | Message a Slack channel when certain events are encountered | | * | | -| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | -| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | -| teams | output | No | Message a Teams channel when certain events are encountered | | * | | -| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | -| websocket | output | No | Output to websockets | | * | | +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | +|-----------------|--------|-----------------|-----------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------|---------------------------| +| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | +| csv | output | No | Output to CSV | | * | | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | +| emails | output | No | Output any email addresses found belonging to the target domain | | EMAIL_ADDRESS | | +| http | output | No | Send every event to a custom URL via a web request | | * | | +| human | output | No | Output to text | | * | | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | +| neo4j | output | No | Output to Neo4j | | * | | +| python | output | No | Output via Python API | | * | | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | +| subdomains | output | No | Output only resolved, in-scope subdomains | | DNS_NAME, DNS_NAME_UNRESOLVED | | +| teams | output | No | Message a Teams channel when certain events are encountered | | * | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | +| websocket | output | No | Output to websockets | | * | | diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index aeceeb361..634b38f58 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -4,7 +4,7 @@ class github_codesearch(github): watched_events = ["DNS_NAME"] produced_events = ["CODE_REPOSITORY", "URL_UNVERIFIED"] - flags = ["passive", "subdomain-enum", "safe", "code-enume"] + flags = ["passive", "subdomain-enum", "safe", "code-enum"] meta = {"description": "Query Github's API for code containing the target domain name", "auth_required": True} options = {"api_key": "", "limit": 100} options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"} diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py index 6464daa2b..f55d7fa02 100644 --- a/bbot/modules/gitlab.py +++ b/bbot/modules/gitlab.py @@ -4,7 +4,7 @@ class gitlab(BaseModule): watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"] produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"] - flags = ["active", "safe"] + flags = ["active", "safe", "code-enum"] meta = {"description": "Detect GitLab instances and query them for repositories"} options = {"api_key": ""} options_desc = {"api_key": "Gitlab access token"} diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index f6f5c8e30..a93acc2e5 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -133,13 +133,13 @@ def __init__( modules = [modules] if isinstance(output_modules, str): output_modules = [output_modules] - self.exclude_modules = exclude_modules if exclude_modules is not None else [] - self.require_flags = require_flags if require_flags is not None else [] - self.exclude_flags = exclude_flags if exclude_flags is not None else [] - self.flags = flags if flags is not None else [] - self.scan_modules = modules if modules is not None else [] - self.output_modules = output_modules if output_modules is not None else [] - self.internal_modules = internal_modules if internal_modules is not None else [] + self.add_excluded_modules(exclude_modules if exclude_modules is not None else []) + self.add_required_flags(require_flags if require_flags is not None else []) + self.add_excluded_flags(exclude_flags if exclude_flags is not None else []) + self.add_scan_modules(modules if modules is not None else []) + self.add_output_modules(output_modules if output_modules is not None else []) + self.add_internal_modules(internal_modules if internal_modules is not None else []) + self.add_flags(flags if flags is not None else []) @property def bbot_home(self): @@ -152,11 +152,11 @@ def merge(self, other): # module dirs # modules + flags # establish requirements / exclusions first - self.exclude_modules = set(self.exclude_modules).union(set(other.exclude_modules)) - self.require_flags = set(self.require_flags).union(set(other.require_flags)) - self.exclude_flags = set(self.exclude_flags).union(set(other.exclude_flags)) + self.add_excluded_modules(other.exclude_modules) + self.add_required_flags(other.require_flags) + self.add_excluded_flags(other.exclude_flags) # then it's okay to start enabling modules - self.flags = set(self.flags).union(set(other.flags)) + self.add_flags(other.flags) for module_name in other.modules: module_type = self.preloaded_module(module_name).get("type", "scan") self.add_module(module_name, module_type=module_type) @@ -213,10 +213,6 @@ def bake(self): return baked_preset def parse_args(self): - - from .args import BBOTArgs - - self._args = BBOTArgs(self) self.merge(self.args.preset_from_args()) @property @@ -236,18 +232,6 @@ def module_dirs(self, module_dirs): def modules(self): return self._modules - @property - def scan_modules(self): - return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "scan"] - - @property - def output_modules(self): - return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "output"] - - @property - def internal_modules(self): - return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] - @modules.setter def modules(self, modules): if isinstance(modules, str): @@ -256,16 +240,31 @@ def modules(self, modules): for module_name in modules: self.add_module(module_name) + @property + def scan_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "scan"] + @scan_modules.setter def scan_modules(self, modules): + self.log_debug(f"Setting scan modules to {modules}") self._modules_setter(modules, module_type="scan") + @property + def output_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "output"] + @output_modules.setter def output_modules(self, modules): + self.log_debug(f"Setting output modules to {modules}") self._modules_setter(modules, module_type="output") + @property + def internal_modules(self): + return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] + @internal_modules.setter def internal_modules(self, modules): + self.log_debug(f"Setting internal modules to {modules}") self._modules_setter(modules, module_type="internal") def _modules_setter(self, modules, module_type="scan"): @@ -278,6 +277,18 @@ def _modules_setter(self, modules, module_type="scan"): for module_name in set(modules): self.add_module(module_name, module_type=module_type) + def add_scan_modules(self, modules): + for module in modules: + self.add_module(module, module_type="scan") + + def add_output_modules(self, modules): + for module in modules: + self.add_module(module, module_type="output") + + def add_internal_modules(self, modules): + for module in modules: + self.add_module(module, module_type="internal") + def add_module(self, module_name, module_type="scan"): # log.info(f'Adding "{module_name}": {module_type}') if module_name in self.exclude_modules: @@ -334,21 +345,26 @@ def flags(self): @flags.setter def flags(self, flags): - log.debug(f"{self.name}: setting flags to {flags}") - if isinstance(flags, str): - flags = [flags] - for flag in flags: - if not flag in self.module_loader._all_flags: - raise EnableFlagError(f'Flag "{flag}" was not found') + self.log_debug(f"Setting flags to {flags}") self._flags = set(flags) - if self._flags: - for module, preloaded in self.module_loader.preloaded().items(): - module_flags = preloaded.get("flags", []) - if any(f in self._flags for f in module_flags): - self.add_module(module) + for flag in flags: + self.add_flag(flag) + + def add_flags(self, flags): + for flag in flags: + self.add_flag(flag) + + def add_flag(self, flag): + if not flag in self.module_loader._all_flags: + raise EnableFlagError(f'Flag "{flag}" was not found') + for module, preloaded in self.module_loader.preloaded().items(): + module_flags = preloaded.get("flags", []) + if flag in module_flags: + self.add_module(module) @require_flags.setter def require_flags(self, flags): + self.log_debug(f"Setting required flags to {flags}") if isinstance(flags, str): flags = [flags] self._require_flags = set() @@ -357,6 +373,7 @@ def require_flags(self, flags): @exclude_modules.setter def exclude_modules(self, modules): + self.log_debug(f"Setting excluded modules to {modules}") if isinstance(modules, str): modules = [modules] self._exclude_modules = set() @@ -365,12 +382,17 @@ def exclude_modules(self, modules): @exclude_flags.setter def exclude_flags(self, flags): + self.log_debug(f"Setting excluded flags to {flags}") if isinstance(flags, str): flags = [flags] self._exclude_flags = set() for flag in set(flags): self.add_excluded_flag(flag) + def add_required_flags(self, flags): + for flag in flags: + self.add_required_flag(flag) + def add_required_flag(self, flag): self.require_flags.add(flag) for module in list(self.scan_modules): @@ -379,6 +401,10 @@ def add_required_flag(self, flag): self.log_verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') self.modules.remove(module) + def add_excluded_flags(self, flags): + for flag in flags: + self.add_excluded_flag(flag) + def add_excluded_flag(self, flag): self.exclude_flags.add(flag) for module in list(self.scan_modules): @@ -387,6 +413,10 @@ def add_excluded_flag(self, flag): self.log_verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') self.modules.remove(module) + def add_excluded_modules(self, modules): + for module in modules: + self.add_excluded_module(module) + def add_excluded_module(self, module): self.exclude_modules.add(module) for module in list(self.scan_modules): @@ -405,14 +435,6 @@ def config(self): def verbose(self): return self._verbose - @property - def debug(self): - return self._debug - - @property - def silent(self): - return self._silent - @verbose.setter def verbose(self, value): if value: @@ -426,6 +448,10 @@ def verbose(self, value): self.core.logger.log_level = "INFO" self._verbose = value + @property + def debug(self): + return self._debug + @debug.setter def debug(self, value): if value: @@ -439,6 +465,10 @@ def debug(self, value): self.core.logger.log_level = "INFO" self._debug = value + @property + def silent(self): + return self._silent + @silent.setter def silent(self, value): if value: @@ -480,6 +510,10 @@ def environ(self): @property def args(self): + if self._args is None: + from .args import BBOTArgs + + self._args = BBOTArgs(self) return self._args def in_scope(self, e): @@ -517,7 +551,7 @@ def whitelisted(self, e): return e in self.whitelist @classmethod - def from_dict(cls, preset_dict, name=None, _exclude=None): + def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False): new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -541,6 +575,7 @@ def from_dict(cls, preset_dict, name=None, _exclude=None): description=preset_dict.get("description"), conditions=preset_dict.get("conditions", []), _exclude=_exclude, + _log=_log, ) return new_preset @@ -553,7 +588,7 @@ def include_preset(self, filename): self._preset_files_loaded.add(preset_filename) @classmethod - def from_yaml_file(cls, filename, _exclude=None): + def from_yaml_file(cls, filename, _exclude=None, _log=False): """ Create a preset from a YAML file. If the full path is not specified, BBOT will look in all the usual places for it. @@ -569,7 +604,7 @@ def from_yaml_file(cls, filename, _exclude=None): _exclude = set(_exclude) _exclude.add(filename) try: - return cls.from_dict(omegaconf.OmegaConf.load(filename), name=filename.stem, _exclude=_exclude) + return cls.from_dict(omegaconf.OmegaConf.load(filename), name=filename.stem, _exclude=_exclude, _log=_log) except FileNotFoundError: raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') @@ -650,29 +685,35 @@ def all_presets(cls): for preset_path in PRESET_PATH: for yaml_file in preset_path.rglob(f"**/*.{ext}"): try: - loaded_preset = cls.from_yaml_file(yaml_file) + loaded_preset = cls.from_yaml_file(yaml_file, _log=True) category = str(yaml_file.relative_to(preset_path).parent) if category == ".": category = "default" - preset_files[yaml_file] = (loaded_preset, category) + preset_files[yaml_file] = (yaml_file, loaded_preset, category) except Exception as e: log.warning(f'Failed to load preset at "{yaml_file}": {e}') log.trace(traceback.format_exc()) continue return preset_files - def presets_table(self): + def presets_table(self, include_modules=True): table = [] - header = ["Preset", "Category", "Description", "Modules"] - for loaded_preset, category in self.all_presets().values(): - modules = ", ".join(sorted(loaded_preset.scan_modules)) - table.append([loaded_preset.name, category, loaded_preset.description, modules]) + header = ["Preset", "Category", "Description", "# Modules"] + if include_modules: + header.append("Modules") + all_presets = sorted(self.all_presets().values(), key=lambda x: x[1].name) + for yaml_file, loaded_preset, category in all_presets: + num_modules = f"{len(loaded_preset.scan_modules):,}" + row = [loaded_preset.name, category, loaded_preset.description, num_modules] + if include_modules: + row.append(", ".join(sorted(loaded_preset.scan_modules))) + table.append(row) return make_table(table, header) def log_verbose(self, msg): if self._log: - log.verbose(f"preset {self.name}: {msg}") + log.verbose(f"Preset {self.name}: {msg}") def log_debug(self, msg): if self._log: - self.log_debug(f"preset {self.name}: {msg}") + log.debug(f"Preset {self.name}: {msg}") diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index e3bacc5ac..effc0cf23 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -5,8 +5,9 @@ import yaml from pathlib import Path -# PRESET TODO: revisit this -from bbot.core import CORE +from bbot.scanner import Preset + +DEFAULT_PRESET = Preset() os.environ["BBOT_TABLE_FORMAT"] = "github" @@ -64,12 +65,12 @@ def update_individual_module_options(): content = f.read() for match in regex.finditer(content): module_name = match.groups()[0].lower() - bbot_module_options_table = CORE.module_loader.modules_options_table(modules=[module_name]) + bbot_module_options_table = DEFAULT_PRESET.module_loader.modules_options_table(modules=[module_name]) find_replace_file(file, f"BBOT MODULE OPTIONS {module_name.upper()}", bbot_module_options_table) # Example commands bbot_example_commands = [] - for title, description, command in CORE.args.scan_examples: + for title, description, command in DEFAULT_PRESET.args.scan_examples: example = "" example += f"**{title}:**\n\n" # example += f"{description}\n" @@ -80,37 +81,43 @@ def update_individual_module_options(): update_md_files("BBOT EXAMPLE COMMANDS", bbot_example_commands) # Help output - bbot_help_output = CORE.args.parser.format_help().replace("docs.py", "bbot") + bbot_help_output = DEFAULT_PRESET.args.parser.format_help().replace("docs.py", "bbot") bbot_help_output = f"```text\n{bbot_help_output}\n```" assert len(bbot_help_output.splitlines()) > 50 update_md_files("BBOT HELP OUTPUT", bbot_help_output) # BBOT events - bbot_event_table = CORE.module_loader.events_table() + bbot_event_table = DEFAULT_PRESET.module_loader.events_table() assert len(bbot_event_table.splitlines()) > 10 update_md_files("BBOT EVENTS", bbot_event_table) # BBOT modules - bbot_module_table = CORE.module_loader.modules_table() + bbot_module_table = DEFAULT_PRESET.module_loader.modules_table() assert len(bbot_module_table.splitlines()) > 50 update_md_files("BBOT MODULES", bbot_module_table) # BBOT output modules - bbot_output_module_table = CORE.module_loader.modules_table(mod_type="output") + bbot_output_module_table = DEFAULT_PRESET.module_loader.modules_table(mod_type="output") assert len(bbot_output_module_table.splitlines()) > 10 update_md_files("BBOT OUTPUT MODULES", bbot_output_module_table) # BBOT module options - bbot_module_options_table = CORE.module_loader.modules_options_table() + bbot_module_options_table = DEFAULT_PRESET.module_loader.modules_options_table() assert len(bbot_module_options_table.splitlines()) > 100 update_md_files("BBOT MODULE OPTIONS", bbot_module_options_table) update_individual_module_options() # BBOT module flags - bbot_module_flags_table = CORE.module_loader.flags_table() + bbot_module_flags_table = DEFAULT_PRESET.module_loader.flags_table() assert len(bbot_module_flags_table.splitlines()) > 10 update_md_files("BBOT MODULE FLAGS", bbot_module_flags_table) + # BBOT presets + preset = Preset() + bbot_presets_table = DEFAULT_PRESET.presets_table(include_modules=True) + assert len(bbot_presets_table.splitlines()) > 5 + update_md_files("BBOT PRESETS", bbot_presets_table) + # Default config default_config_file = bbot_code_dir / "bbot" / "defaults.yml" with open(default_config_file) as f: diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index e646864e8..1a268fca0 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -14,7 +14,7 @@ | bucket_google | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | | bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | | dastardly | scan | No | Lightweight web application security scanner | active, aggressive, deadly, slow, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | -| dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | active, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | +| dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | active, code-enum, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | | dotnetnuke | scan | No | Scan for critical DotNetNuke (DNN) vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE | TECHNOLOGY, VULNERABILITY | | ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | | ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | @@ -70,8 +70,8 @@ | emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | | fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | | git_clone | scan | No | Clone code github repositories | passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | -| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | -| github_org | scan | No | Query Github's API for organization and member repositories | passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | +| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enume, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | +| github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | | hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | | hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | | internetdb | scan | No | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities | passive, portscan, safe, subdomain-enum | DNS_NAME, IP_ADDRESS | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY | @@ -104,7 +104,7 @@ | asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | | csv | output | No | Output to CSV | | * | | | discord | output | No | Message a Discord channel when certain events are encountered | | * | | -| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | +| emails | output | No | Output any email addresses found belonging to the target domain | | EMAIL_ADDRESS | | | http | output | No | Send every event to a custom URL via a web request | | * | | | human | output | No | Output to text | | * | | | json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | @@ -112,7 +112,7 @@ | python | output | No | Output via Python API | | * | | | slack | output | No | Message a Slack channel when certain events are encountered | | * | | | splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | -| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | +| subdomains | output | No | Output only resolved, in-scope subdomains | | DNS_NAME, DNS_NAME_UNRESOLVED | | | teams | output | No | Message a Teams channel when certain events are encountered | | * | | | web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | | websocket | output | No | Output to websockets | | * | | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 71aa658f4..0a41753b8 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -33,22 +33,15 @@ asyncio.run(main()) ```text -usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] - [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] - [--strict-scope] [-m MODULE [MODULE ...]] [-l] - [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] - [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] - [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] - [-o DIR] [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] - [--dry-run] [--current-config] - [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] - [-a] [--version] +usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-p [PRESET ...]] [-c [CONFIG ...]] [-lp] + [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] + [--allow-deadly] [-n SCAN_NAME] [-o DIR] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-preset] [--current-preset-full] + [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version] Bighuge BLS OSINT Tool options: -h, --help show this help message and exit - --help-all Display full help including module config options Target: -t TARGET [TARGET ...], --targets TARGET [TARGET ...] @@ -59,14 +52,23 @@ Target: Don't touch these things --strict-scope Don't consider subdomains of target/whitelist to be in-scope +Presets: + -p [PRESET ...], --preset [PRESET ...] + Enable BBOT preset(s) + -c [CONFIG ...], --config [CONFIG ...] + Custom config options in key=value format: e.g. 'modules.shodan.api_key=1234' + -lp, --list-presets List available presets. + Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,baddns,baddns_zone,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,dockerhub,dotnetnuke,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,github_codesearch,github_org,gitlab,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,sublist3r,telerik,threatminer,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye -l, --list-modules List available modules. + -lmo, --list-module-options + Show all module config options -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. -f FLAG [FLAG ...], --flags FLAG [FLAG ...] - Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough + Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,code-enum,code-enume,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough -lf, --list-flags List available flags. -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] Only enable modules with these flags (e.g. -rf passive) @@ -80,15 +82,15 @@ Scan: -n SCAN_NAME, --name SCAN_NAME Name of scan (default: random) -o DIR, --output-dir DIR - -c [CONFIG ...], --config [CONFIG ...] - custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234' -v, --verbose Be more verbose -d, --debug Enable debugging -s, --silent Be quiet - --force Run scan even if module setups fail + --force Run scan even in the case of condition violations or failed module setups -y, --yes Skip scan confirmation prompt --dry-run Abort before executing scan - --current-config Show current config in YAML format + --current-preset Show the current preset in YAML format + --current-preset-full + Show the current preset in its full form, including defaults Module dependencies: Control how modules install their dependencies @@ -99,11 +101,6 @@ Module dependencies: --ignore-failed-deps Run modules even if they have failed dependencies --install-all-deps Install dependencies for all modules -Agent: - Report back to a central server - - -a, --agent-mode Start in agent mode - Misc: --version show BBOT version and exit @@ -130,6 +127,9 @@ EXAMPLES List modules: bbot -l + List presets: + bbot -lp + List flags: bbot -lf diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index fba53cd3d..db018db58 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -90,6 +90,8 @@ web_spider_links_per_page: 25 ### ADVANCED OPTIONS ### +module_paths: [] + # How far out from the main scope to search scope_search_distance: 0 # How far out from the main scope to resolve DNS names / IPs @@ -104,6 +106,14 @@ excavate: True # Summarize activity at the end of a scan aggregate: True +# How to handle installation of module dependencies +# Choices are: +# - abort_on_failure (default) - if a module dependency fails to install, abort the scan +# - retry_failed - try again to install failed dependencies +# - ignore_failed - run the scan regardless of what happens with dependency installation +# - disable - completely disable BBOT's dependency system (you are responsible for install tools, pip packages, etc.) +deps_behavior: abort_on_failure + # HTTP timeout (for Python requests; API calls, etc.) http_timeout: 10 # HTTP timeout (for httpx) @@ -181,10 +191,6 @@ omit_event_types: - URL_UNVERIFIED - DNS_NAME_UNRESOLVED # - IP_ADDRESS -# URL of BBOT server -agent_url: '' -# Agent Bearer authentication token -agent_token: '' # Custom interactsh server settings interactsh_server: null @@ -351,48 +357,48 @@ Many modules accept their own configuration options. These options have the abil | modules.zoomeye.api_key | str | ZoomEye API key | | | modules.zoomeye.include_related | bool | Include domains which may be related to the target | False | | modules.zoomeye.max_pages | int | How many pages of results to fetch | 20 | -| output_modules.asset_inventory.output_file | str | Set a custom output file | | -| output_modules.asset_inventory.recheck | bool | When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan | False | -| output_modules.asset_inventory.summary_netmask | int | Subnet mask to use when summarizing IP addresses at end of scan | 16 | -| output_modules.asset_inventory.use_previous | bool |` Emit previous asset inventory as new events (use in conjunction with -n ) `| False | -| output_modules.csv.output_file | str | Output to CSV file | | -| output_modules.discord.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.discord.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | -| output_modules.discord.webhook_url | str | Discord webhook URL | | -| output_modules.emails.output_file | str | Output to file | | -| output_modules.http.bearer | str | Authorization Bearer token | | -| output_modules.http.method | str | HTTP method | POST | -| output_modules.http.password | str | Password (basic auth) | | -| output_modules.http.siem_friendly | bool | Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc. | False | -| output_modules.http.timeout | int | HTTP timeout | 10 | -| output_modules.http.url | str | Web URL | | -| output_modules.http.username | str | Username (basic auth) | | -| output_modules.human.console | bool | Output to console | True | -| output_modules.human.output_file | str | Output to file | | -| output_modules.json.console | bool | Output to console | False | -| output_modules.json.output_file | str | Output to file | | -| output_modules.json.siem_friendly | bool | Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc. | False | -| output_modules.neo4j.password | str | Neo4j password | bbotislife | -| output_modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 | -| output_modules.neo4j.username | str | Neo4j username | neo4j | -| output_modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | -| output_modules.slack.webhook_url | str | Discord webhook URL | | -| output_modules.splunk.hectoken | str | HEC Token | | -| output_modules.splunk.index | str | Index to send data to | | -| output_modules.splunk.source | str | Source path to be added to the metadata | | -| output_modules.splunk.timeout | int | HTTP timeout | 10 | -| output_modules.splunk.url | str | Web URL | | -| output_modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False | -| output_modules.subdomains.output_file | str | Output to file | | -| output_modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | -| output_modules.teams.webhook_url | str | Discord webhook URL | | -| output_modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css | -| output_modules.web_report.output_file | str | Output to file | | -| output_modules.websocket.preserve_graph | bool | Preserve full chains of events in the graph (prevents orphans) | True | -| output_modules.websocket.token | str | Authorization Bearer token | | -| output_modules.websocket.url | str | Web URL | | -| internal_modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 | -| internal_modules.speculate.ports | str | The set of ports to speculate on | 80,443 | +| modules.asset_inventory.output_file | str | Set a custom output file | | +| modules.asset_inventory.recheck | bool | When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan | False | +| modules.asset_inventory.summary_netmask | int | Subnet mask to use when summarizing IP addresses at end of scan | 16 | +| modules.asset_inventory.use_previous | bool |` Emit previous asset inventory as new events (use in conjunction with -n ) `| False | +| modules.csv.output_file | str | Output to CSV file | | +| modules.discord.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| modules.discord.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | +| modules.discord.webhook_url | str | Discord webhook URL | | +| modules.emails.output_file | str | Output to file | | +| modules.http.bearer | str | Authorization Bearer token | | +| modules.http.method | str | HTTP method | POST | +| modules.http.password | str | Password (basic auth) | | +| modules.http.siem_friendly | bool | Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc. | False | +| modules.http.timeout | int | HTTP timeout | 10 | +| modules.http.url | str | Web URL | | +| modules.http.username | str | Username (basic auth) | | +| modules.human.console | bool | Output to console | True | +| modules.human.output_file | str | Output to file | | +| modules.json.console | bool | Output to console | False | +| modules.json.output_file | str | Output to file | | +| modules.json.siem_friendly | bool | Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc. | False | +| modules.neo4j.password | str | Neo4j password | bbotislife | +| modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 | +| modules.neo4j.username | str | Neo4j username | neo4j | +| modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | +| modules.slack.webhook_url | str | Discord webhook URL | | +| modules.splunk.hectoken | str | HEC Token | | +| modules.splunk.index | str | Index to send data to | | +| modules.splunk.source | str | Source path to be added to the metadata | | +| modules.splunk.timeout | int | HTTP timeout | 10 | +| modules.splunk.url | str | Web URL | | +| modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False | +| modules.subdomains.output_file | str | Output to file | | +| modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | +| modules.teams.webhook_url | str | Discord webhook URL | | +| modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css | +| modules.web_report.output_file | str | Output to file | | +| modules.websocket.preserve_graph | bool | Preserve full chains of events in the graph (prevents orphans) | True | +| modules.websocket.token | str | Authorization Bearer token | | +| modules.websocket.url | str | Web URL | | +| modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 | +| modules.speculate.ports | str | The set of ports to speculate on | 80,443 | diff --git a/docs/scanning/index.md b/docs/scanning/index.md index c2863ec23..706256a58 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -111,7 +111,7 @@ A single module can have multiple flags. For example, the `securitytrails` modul |------------------|-------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | safe | 79 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | | passive | 58 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 45 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | +| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | | active | 43 | Makes active connections to target systems | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dockerhub, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer | | web-thorough | 29 | More advanced web scanning functionality | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | | aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | @@ -119,14 +119,16 @@ A single module can have multiple flags. For example, the `securitytrails` modul | cloud-enum | 12 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth | | slow | 9 | May take a long time to complete | bucket_digitalocean, dastardly, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | | affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | -| email-enum | 7 | Enumerates email addresses | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert | +| email-enum | 6 | Enumerates email addresses | dehashed, emailformat, hunterio, pgp, skymem, sslcert | | deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | | portscan | 3 | Discovers open ports | internetdb, masscan, nmap | | web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | | baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | +| code-enum | 2 | | dockerhub, github_org | | iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | | report | 2 | Generates a report at the end of the scan | affiliates, asn | | social-enum | 2 | Enumerates social media | httpx, social | +| code-enume | 1 | | github_codesearch | | service-enum | 1 | Identifies protocols running on open ports | fingerprintx | | subdomain-hijack | 1 | Detects hijackable subdomains | baddns | | web-screenshots | 1 | Takes screenshots of web pages | gowitness | diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md new file mode 100644 index 000000000..a689773c1 --- /dev/null +++ b/docs/scanning/presets.md @@ -0,0 +1,188 @@ +# Presets + +A Preset is a YAML file where you can put all the settings for a scan. That include targets, modules, and config options like API keys. + +## Default Presets + +BBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`. You can enable them like this: + +```bash +# do a subdomain enumeration +bbot -t evilcorp.com -p subdomain-enum + +# multiple presets - subdomain enumeration + web spider +bbot -t evilcorp.com -p subdomain-enum spider + +# start with a preset but only enable modules that have the 'passive' flag +bbot -t evilcorp.com -p subdomain-enum -rf passive + +# preset + manual config override +bbot -t www.evilcorp.com -p spider -c web_spider_distance=10 + +# list available presets +bbot -lp +``` + +You can also build on the default presets, or create your own. Here's an example of a custom preset that builds on `subdomain-enum`: + +```yaml title="my_subdomains.yml" +description: Do a subdomain enumeration + basic web scan + nuclei + +target: + - evilcorp.com + +include: + - subdomain-enum + - web-basic + +modules: + # enable nuclei in addition to the other modules + - nuclei + +config: + modules: + securitytrails: + api_key: 21a270d5f59c9b05813a72bb41707266 + virustotal: + api_key: 4f41243847da693a4f356c0486114bc6 +``` + +To execute your custom preset, you do: + +```bash +bbot -p ./my_subdomains.yml +``` + +## Preset Load Order + +When you enable multiple presets, the order matters. In the case of a conflict, the last preset will always win. This means, for example, if you have a custom preset called `my_spider` that sets `web_spider_distance` to 1: + +```yaml title="my_spider.yml" +web_spider_distance: 1 +``` + +...and you enable it alongside the default `spider` preset in this order: + +```bash +bbot -t evilcorp.com -p ./my_spider.yml spider +``` + +...the value of `web_spider_distance` will be overridden by `spider`. To ensure this doesn't happen, you would want to switch the order of the presets: + +```bash +bbot -t evilcorp.com -p spider ./my_spider.yml +``` + +## Validating Presets + +To make sure BBOT is configured the way you expect, you can always check the `--current-preset` to show the final verison of the config that will be used when BBOT executes: + +```bash +# verify the preset is what you want +bbot -p ./mypreset.yml --current-preset +``` + +## Advanced Usage + +BBOT Presets support advanced features like environment variable substitution and custom conditions. + +### Environment Variables + +You can insert environment variables into your preset like this: `${env:}`: + +```yaml title="my_nuclei.yml" +description: Do a nuclei scan + +target: + - evilcorp.com + +modules: + - nuclei + +config: + modules: + nuclei: + # allow the nuclei templates to be specified at runtime via an environment variable + templates: ${env:NUCLEI_TEMPLATES} +``` + +```bash +NUCLEI_TEMPLATES=apache,nginx bbot -p ./my_nuclei.yml +``` + +### Conditions + +Sometimes, you might need to add custom logic to a preset. BBOT supports this via `conditions`. The `conditions` attribute allows you to specify a list of custom conditions that will be evaluated before the scan starts. This is useful for performing last-minute sanity checks, or changing the behavior of the scan based on custom criteria. + +```yaml title="my_preset.yml" +description: Abort if nuclei templates aren't specified + +modules: + - nuclei + +conditions: + - | + {% if not config.modules.nuclei.templates %} + {{ abort("Don't forget to set your templates!") }} + {% endif %} +``` + +```yaml title="my_preset.yml" +description: Enable ffuf but only if the web spider isn't enabled + +modules: + - ffuf + +conditions: + - | + {% if config.web_spider_distance > 0 and config.web_spider_depth > 0 %} + {{ warn("Disabling ffuf because the web spider is enabled") }} + {{ preset.exclude_module("ffuf") }} + {% endif %} +``` + +Conditions use [Jinja](https://palletsprojects.com/p/jinja/), which means they can contain Python code. They run inside a sandboxed environment which has access to the following variables: + +- `preset` - the current preset object +- `config` - the current config (an alias for `preset.config`) +- `warn(message)` - display a custom warning message to the user +- `abort(message)` - abort the scan with an optional message + +If you aren't able to accomplish what you want with conditions, or if you need access to a new variable/function, please let us know on [Github](https://github.com/blacklanternsecurity/bbot/issues/new/choose). + + +## List of Presets + +Here is a full list of BBOT's default presets: + + +| Preset | Category | Description | # Modules | Modules | +|----------------|--------------|------------------------------------------------------------------|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| cloud-enum | default | Enumerate cloud resources such as storage buckets, etc. | 52 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| code-enum | default | Enumerate Git repositories, Docker images, etc. | 3 | dockerhub, github_org, social | +| dirbust-heavy | web_advanced | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | +| dirbust-light | web_advanced | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | +| email-enum | default | Enumerate email addresses from APIs, web crawling, etc. | 6 | dehashed, emailformat, hunterio, pgp, skymem, sslcert | +| kitchen-sink | default | Everything everywhere all at once | 70 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, massdns, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wappalyzer, wayback, zoomeye | +| paramminer | web_advanced | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | +| secrets-enum | default | | 0 | | +| spider | default | Recursive web spider | 1 | httpx | +| subdomain-enum | default | Enumerate subdomains via APIs, brute-force | 45 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| web-basic | default | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | +| web-thorough | default | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | + + +
`subdomain-enum` + +```yaml title="~/.bbot/presets/subdomain-enum.yml" +description: Enumerate subdomains via APIs, brute-force + +flags: + - subdomain-enum + +output_modules: + - subdomains + +``` + +
From 08e68c5dfd2d507e76922f0db9a67f99374452e4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 25 Mar 2024 17:30:06 -0400 Subject: [PATCH 055/171] steady work on presets --- bbot/core/core.py | 9 ++------- bbot/core/helpers/misc.py | 4 ++-- bbot/presets/subdomain-enum.yml | 2 -- bbot/scanner/preset/environ.py | 10 ++++++++++ bbot/scanner/preset/preset.py | 20 +++++++++++--------- bbot/scripts/docs.py | 1 - docs/scanning/configuration.md | 4 +++- docs/scanning/presets.md | 15 --------------- mkdocs.yml | 1 + 9 files changed, 29 insertions(+), 37 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index 25dabeece..e8170c4eb 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -25,10 +25,6 @@ def __init__(self): self.logger self.log = logging.getLogger("bbot.core") - # PRESET TODO: add back in bbot/core/configurator/__init__.py - # - check_cli_args - # - ensure_config_files - @property def home(self): return Path(self.config["home"]).expanduser().resolve() @@ -70,9 +66,7 @@ def config(self): def default_config(self): global DEFAULT_CONFIG if DEFAULT_CONFIG is None: - DEFAULT_CONFIG = self.files_config.get_default_config() - # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(DEFAULT_CONFIG, True) + self.default_config = self.files_config.get_default_config() return DEFAULT_CONFIG @default_config.setter @@ -81,6 +75,7 @@ def default_config(self, value): global DEFAULT_CONFIG self._config = None DEFAULT_CONFIG = value + # set read-only flag (change .custom_config instead) OmegaConf.set_readonly(DEFAULT_CONFIG, True) @property diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 77b88310b..1ae26a425 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2007,8 +2007,8 @@ def make_table(rows, header, *args, **kwargs): from tabulate import tabulate # fix IndexError: list index out of range - if args and not args[0]: - args = ([[]],) + args[1:] + if not rows: + rows = [[]] tablefmt = os.environ.get("BBOT_TABLE_FORMAT", None) defaults = {"tablefmt": "grid", "disable_numparse": True, "maxcolwidths": None} if tablefmt is None: diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index 24f9d1dac..1741a06e5 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -8,8 +8,6 @@ output_modules: config: modules: - c99: - myoption: "#{MYOPTION}" stdout: format: text # only output DNS_NAMEs to the console diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 21f951bd3..9bf2ad49b 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -24,6 +24,16 @@ def increase_limit(new_limit): increase_limit(65535) +# Custom custom omegaconf resolver to get environment variables +def env_resolver(env_name, default=None): + return os.getenv(env_name, default) + + +# Register the new resolver +# this allows you to substitute environment variables in your config like "${env:PATH}"" +omegaconf.OmegaConf.register_new_resolver("env", env_resolver) + + class BBOTEnviron: def __init__(self, preset): diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index a93acc2e5..11b99dabd 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -198,6 +198,8 @@ def bake(self): # prepare os environment os_environ = baked_preset.environ.prepare() # find and replace preloaded modules with os environ + # this is different from the config variable substitution because it modifies + # the preloaded modules, i.e. their ansible playbooks baked_preset.module_loader.find_and_replace(**os_environ) # update os environ os.environ.clear() @@ -369,7 +371,7 @@ def require_flags(self, flags): flags = [flags] self._require_flags = set() for flag in set(flags): - self.add_required_flag(flag) + self.require_flag(flag) @exclude_modules.setter def exclude_modules(self, modules): @@ -378,7 +380,7 @@ def exclude_modules(self, modules): modules = [modules] self._exclude_modules = set() for module in set(modules): - self.add_excluded_module(module) + self.exclude_module(module) @exclude_flags.setter def exclude_flags(self, flags): @@ -387,13 +389,13 @@ def exclude_flags(self, flags): flags = [flags] self._exclude_flags = set() for flag in set(flags): - self.add_excluded_flag(flag) + self.exclude_flag(flag) def add_required_flags(self, flags): for flag in flags: - self.add_required_flag(flag) + self.require_flag(flag) - def add_required_flag(self, flag): + def require_flag(self, flag): self.require_flags.add(flag) for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) @@ -403,9 +405,9 @@ def add_required_flag(self, flag): def add_excluded_flags(self, flags): for flag in flags: - self.add_excluded_flag(flag) + self.exclude_flag(flag) - def add_excluded_flag(self, flag): + def exclude_flag(self, flag): self.exclude_flags.add(flag) for module in list(self.scan_modules): module_flags = self.preloaded_module(module).get("flags", []) @@ -415,9 +417,9 @@ def add_excluded_flag(self, flag): def add_excluded_modules(self, modules): for module in modules: - self.add_excluded_module(module) + self.exclude_module(module) - def add_excluded_module(self, module): + def exclude_module(self, module): self.exclude_modules.add(module) for module in list(self.scan_modules): if module in self.exclude_modules: diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index effc0cf23..33e1e47b9 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -113,7 +113,6 @@ def update_individual_module_options(): update_md_files("BBOT MODULE FLAGS", bbot_module_flags_table) # BBOT presets - preset = Preset() bbot_presets_table = DEFAULT_PRESET.presets_table(include_modules=True) assert len(bbot_presets_table.splitlines()) > 5 update_md_files("BBOT PRESETS", bbot_presets_table) diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index db018db58..5e0241057 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -1,6 +1,8 @@ # Configuration Overview -BBOT has a YAML config at `~/.config/bbot`. This config is different from the command-line arguments. This is where you change settings such as BBOT's **HTTP proxy**, **rate limits**, or global **User-Agent**. It's also where you put modules' **API keys**. +Normally, [Presets](presets.md) are used to configure a scan. However, there may be cases where you want to change BBOT's defaults so a certain option is always set, even if it's not specified in a preset. + +BBOT has a YAML config at `~/.config/bbot.yml`. This is the first config that BBOT loads, so it's a good place to put default settings like `http_proxy`, `max_threads`, or `http_user_agent`. You can also put any module settings here, including **API keys**. For a list of all possible config options, see: diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index a689773c1..b331dec1c 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -171,18 +171,3 @@ Here is a full list of BBOT's default presets: | web-basic | default | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | | web-thorough | default | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | - -
`subdomain-enum` - -```yaml title="~/.bbot/presets/subdomain-enum.yml" -description: Enumerate subdomains via APIs, brute-force - -flags: - - subdomain-enum - -output_modules: - - subdomains - -``` - -
diff --git a/mkdocs.yml b/mkdocs.yml index d7bb10118..62a1b4e95 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,6 +21,7 @@ nav: - Comparison to Other Tools: comparison.md - Scanning: - Scanning Overview: scanning/index.md + - Presets: scanning/presets.md - Events: scanning/events.md - Output: scanning/output.md - Tips and Tricks: scanning/tips_and_tricks.md From 60aa0daed5bef49d8284f281ab74c5f8265bcbaf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 12:19:58 -0400 Subject: [PATCH 056/171] steady work on docs --- README.md | 6 +- bbot/scanner/preset/args.py | 12 +- bbot/scanner/preset/preset.py | 18 +- bbot/scripts/docs.py | 45 +++++ docs/modules/list_of_modules.md | 4 +- docs/scanning/advanced.md | 2 +- docs/scanning/index.md | 3 +- docs/scanning/presets.md | 78 ++++++-- docs/scanning/presets_list.md | 338 ++++++++++++++++++++++++++++++++ mkdocs.yml | 5 +- 10 files changed, 470 insertions(+), 41 deletions(-) create mode 100644 docs/scanning/presets_list.md diff --git a/README.md b/README.md index 876ddf5c2..7b6922b35 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,9 @@ asyncio.run(main()) - [Comparison to Other Tools](https://www.blacklanternsecurity.com/bbot/comparison) - **Scanning** - [Scanning Overview](https://www.blacklanternsecurity.com/bbot/scanning/) + - **Presets** + - [Overview](https://www.blacklanternsecurity.com/bbot/scanning/presets) + - [List of Presets](https://www.blacklanternsecurity.com/bbot/scanning/presets_list) - [Events](https://www.blacklanternsecurity.com/bbot/scanning/events) - [Output](https://www.blacklanternsecurity.com/bbot/scanning/output) - [Tips and Tricks](https://www.blacklanternsecurity.com/bbot/scanning/tips_and_tricks) @@ -277,15 +280,14 @@ For a full list of modules, including the data types consumed and emitted by eac | slow | 9 | May take a long time to complete | bucket_digitalocean, dastardly, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | | affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | | email-enum | 6 | Enumerates email addresses | dehashed, emailformat, hunterio, pgp, skymem, sslcert | +| code-enum | 4 | | dockerhub, github_codesearch, github_org, gitlab | | deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | | portscan | 3 | Discovers open ports | internetdb, masscan, nmap | | web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | | baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | -| code-enum | 2 | | dockerhub, github_org | | iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | | report | 2 | Generates a report at the end of the scan | affiliates, asn | | social-enum | 2 | Enumerates social media | httpx, social | -| code-enume | 1 | | github_codesearch | | service-enum | 1 | Identifies protocols running on open ports | fingerprintx | | subdomain-hijack | 1 | Detects hijackable subdomains | baddns | | web-screenshots | 1 | Takes screenshots of web pages | gowitness | diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 5c9d8202c..0e24dd0d0 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -18,32 +18,32 @@ class BBOTArgs: ( "Subdomains", "Perform a full subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum", + "bbot -t evilcorp.com -p subdomain-enum", ), ( "Subdomains (passive only)", "Perform a passive-only subdomain enumeration on evilcorp.com", - "bbot -t evilcorp.com -f subdomain-enum -rf passive", + "bbot -t evilcorp.com -p subdomain-enum -rf passive", ), ( "Subdomains + port scan + web screenshots", "Port-scan every subdomain, screenshot every webpage, output to current directory", - "bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o .", + "bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o .", ), ( "Subdomains + basic web scan", "A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules", - "bbot -t evilcorp.com -f subdomain-enum web-basic", + "bbot -t evilcorp.com -p subdomain-enum web-basic", ), ( "Web spider", "Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.", - "bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2", + "bbot -t www.evilcorp.com -p spider -c web_spider_distance=2 web_spider_depth=2", ), ( "Everything everywhere all at once", "Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei", - "bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly", + "bbot -t evilcorp.com -p kitchen-sink", ), ] diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 11b99dabd..b72cd8652 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -54,6 +54,7 @@ def __init__( self._environ = None self._helpers = None self._module_loader = None + self._yaml_str = "" self._modules = set() @@ -606,9 +607,12 @@ def from_yaml_file(cls, filename, _exclude=None, _log=False): _exclude = set(_exclude) _exclude.add(filename) try: - return cls.from_dict(omegaconf.OmegaConf.load(filename), name=filename.stem, _exclude=_exclude, _log=_log) + yaml_str = open(filename).read() except FileNotFoundError: raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') + preset = cls.from_dict(omegaconf.OmegaConf.create(yaml_str), name=filename.stem, _exclude=_exclude, _log=_log) + preset._yaml_str = yaml_str + return preset @classmethod def from_yaml_string(cls, yaml_preset): @@ -682,7 +686,7 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): @classmethod def all_presets(cls): - preset_files = dict() + presets = dict() for ext in ("yml", "yaml"): for preset_path in PRESET_PATH: for yaml_file in preset_path.rglob(f"**/*.{ext}"): @@ -690,21 +694,21 @@ def all_presets(cls): loaded_preset = cls.from_yaml_file(yaml_file, _log=True) category = str(yaml_file.relative_to(preset_path).parent) if category == ".": - category = "default" - preset_files[yaml_file] = (yaml_file, loaded_preset, category) + category = "" + presets[yaml_file] = (loaded_preset, category) except Exception as e: log.warning(f'Failed to load preset at "{yaml_file}": {e}') log.trace(traceback.format_exc()) continue - return preset_files + # sort by name + return dict(sorted(presets.items(), key=lambda x: x[-1][0].name)) def presets_table(self, include_modules=True): table = [] header = ["Preset", "Category", "Description", "# Modules"] if include_modules: header.append("Modules") - all_presets = sorted(self.all_presets().values(), key=lambda x: x[1].name) - for yaml_file, loaded_preset, category in all_presets: + for yaml_file, (loaded_preset, category) in self.all_presets().items(): num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] if include_modules: diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index 33e1e47b9..84546f936 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -19,6 +19,14 @@ bbot_code_dir = Path(__file__).parent.parent.parent +def homedir_collapseuser(f): + f = Path(f) + home_dir = Path.home() + if f.is_relative_to(home_dir): + return Path("~") / f.relative_to(home_dir) + return f + + def enclose_tags(text): # Use re.sub() to replace matched words with the same words enclosed in backticks result = blacklist_re.sub(r"|`\1`|", text) @@ -117,6 +125,43 @@ def update_individual_module_options(): assert len(bbot_presets_table.splitlines()) > 5 update_md_files("BBOT PRESETS", bbot_presets_table) + # BBOT subdomain enum preset + for yaml_file, (loaded_preset, category) in DEFAULT_PRESET.all_presets().items(): + if loaded_preset.name == "subdomain-enum": + subdomain_enum_preset = f"""```yaml title="{yaml_file.name}" +{loaded_preset._yaml_str} +```""" + update_md_files("BBOT SUBDOMAIN ENUM PRESET", subdomain_enum_preset) + break + + content = [] + for yaml_file, (loaded_preset, category) in DEFAULT_PRESET.all_presets().items(): + yaml_str = loaded_preset._yaml_str + indent = " " * 4 + yaml_str = f"\n{indent}".join(yaml_str.splitlines()) + filename = homedir_collapseuser(yaml_file) + + num_modules = len(loaded_preset.scan_modules) + modules = ", ".join(sorted([f"`{m}`" for m in loaded_preset.scan_modules])) + category = f"Category: {category}" if category else "" + + content.append( + f"""## **{loaded_preset.name}** + +{loaded_preset.description} + +??? note "`{filename.name}`" + ```yaml title="{filename}" + {yaml_str} + ``` + +{category} + +Modules: [{num_modules:,}]("{modules}")""" + ) + assert len(content) > 5 + update_md_files("BBOT PRESET YAML", "\n\n".join(content)) + # Default config default_config_file = bbot_code_dir / "bbot" / "defaults.yml" with open(default_config_file) as f: diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 1a268fca0..20acd9dd6 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -22,7 +22,7 @@ | fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | | generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | | git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | -| gitlab | scan | No | Detect GitLab instances and query them for repositories | active, safe | HTTP_RESPONSE, SOCIAL, TECHNOLOGY | CODE_REPOSITORY, FINDING, SOCIAL, TECHNOLOGY | +| gitlab | scan | No | Detect GitLab instances and query them for repositories | active, code-enum, safe | HTTP_RESPONSE, SOCIAL, TECHNOLOGY | CODE_REPOSITORY, FINDING, SOCIAL, TECHNOLOGY | | gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | SOCIAL, URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | | host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | | httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | @@ -70,7 +70,7 @@ | emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | | fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | | git_clone | scan | No | Clone code github repositories | passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | -| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enume, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | +| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | | github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | | hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | | hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 0a41753b8..8de312b86 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -68,7 +68,7 @@ Modules: -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. -f FLAG [FLAG ...], --flags FLAG [FLAG ...] - Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,code-enum,code-enume,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough + Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,code-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough -lf, --list-flags List available flags. -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] Only enable modules with these flags (e.g. -rf passive) diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 706256a58..2b591c951 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -120,15 +120,14 @@ A single module can have multiple flags. For example, the `securitytrails` modul | slow | 9 | May take a long time to complete | bucket_digitalocean, dastardly, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | | affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | | email-enum | 6 | Enumerates email addresses | dehashed, emailformat, hunterio, pgp, skymem, sslcert | +| code-enum | 4 | | dockerhub, github_codesearch, github_org, gitlab | | deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | | portscan | 3 | Discovers open ports | internetdb, masscan, nmap | | web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | | baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | -| code-enum | 2 | | dockerhub, github_org | | iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | | report | 2 | Generates a report at the end of the scan | affiliates, asn | | social-enum | 2 | Enumerates social media | httpx, social | -| code-enume | 1 | | github_codesearch | | service-enum | 1 | Identifies protocols running on open ports | fingerprintx | | subdomain-hijack | 1 | Detects hijackable subdomains | baddns | | web-screenshots | 1 | Takes screenshots of web pages | gowitness | diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index b331dec1c..854fa8064 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -1,10 +1,40 @@ # Presets -A Preset is a YAML file where you can put all the settings for a scan. That include targets, modules, and config options like API keys. +Presets allow you to put all of your scan settings in one place. A Preset is a YAML file that can include scan targets, modules, and config options like API keys. -## Default Presets +A typical preset looks like this: -BBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`. You can enable them like this: + +```yaml title="subdomain-enum.yml" +description: Enumerate subdomains via APIs, brute-force + +flags: + - subdomain-enum + +output_modules: + - subdomains + +config: + modules: + stdout: + format: text + # only output DNS_NAMEs to the console + event_types: + - DNS_NAME + # only show in-scope subdomains + in_scope_only: True + # display the raw subdomains, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False + +``` + + +## How to use Presets (`-p`) + +BBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`. You can use them like this: ```bash # do a subdomain enumeration @@ -32,6 +62,7 @@ target: - evilcorp.com include: + # include these default presets - subdomain-enum - web-basic @@ -40,11 +71,18 @@ modules: - nuclei config: + # global config options + http_proxy: http://127.0.0.1:8080 + # module config options modules: + # api keys securitytrails: api_key: 21a270d5f59c9b05813a72bb41707266 virustotal: api_key: 4f41243847da693a4f356c0486114bc6 + # other module config options + massdns: + max_resolvers: 5000 ``` To execute your custom preset, you do: @@ -103,11 +141,11 @@ config: modules: nuclei: # allow the nuclei templates to be specified at runtime via an environment variable - templates: ${env:NUCLEI_TEMPLATES} + tags: ${env:NUCLEI_TAGS} ``` ```bash -NUCLEI_TEMPLATES=apache,nginx bbot -p ./my_nuclei.yml +NUCLEI_TAGS=apache,nginx bbot -p ./my_nuclei.yml ``` ### Conditions @@ -128,7 +166,7 @@ conditions: ``` ```yaml title="my_preset.yml" -description: Enable ffuf but only if the web spider isn't enabled +description: Enable ffuf but only when the web spider isn't also enabled modules: - ffuf @@ -156,18 +194,18 @@ If you aren't able to accomplish what you want with conditions, or if you need a Here is a full list of BBOT's default presets: -| Preset | Category | Description | # Modules | Modules | -|----------------|--------------|------------------------------------------------------------------|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| cloud-enum | default | Enumerate cloud resources such as storage buckets, etc. | 52 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | -| code-enum | default | Enumerate Git repositories, Docker images, etc. | 3 | dockerhub, github_org, social | -| dirbust-heavy | web_advanced | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | -| dirbust-light | web_advanced | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | -| email-enum | default | Enumerate email addresses from APIs, web crawling, etc. | 6 | dehashed, emailformat, hunterio, pgp, skymem, sslcert | -| kitchen-sink | default | Everything everywhere all at once | 70 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, massdns, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wappalyzer, wayback, zoomeye | -| paramminer | web_advanced | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | -| secrets-enum | default | | 0 | | -| spider | default | Recursive web spider | 1 | httpx | -| subdomain-enum | default | Enumerate subdomains via APIs, brute-force | 45 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | -| web-basic | default | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | -| web-thorough | default | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | +| Preset | Category | Description | # Modules | Modules | +|----------------|--------------|------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 52 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| code-enum | | Enumerate Git repositories, Docker images, etc. | 6 | dockerhub, github_codesearch, github_org, gitlab, httpx, social | +| dirbust-heavy | web_advanced | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | +| dirbust-light | web_advanced | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | +| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 6 | dehashed, emailformat, hunterio, pgp, skymem, sslcert | +| kitchen-sink | | Everything everywhere all at once | 71 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, gitlab, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, massdns, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wappalyzer, wayback, zoomeye | +| paramminer | web_advanced | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | +| secrets-enum | | | 0 | | +| spider | | Recursive web spider | 1 | httpx | +| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 45 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| web-basic | | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | +| web-thorough | | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | diff --git a/docs/scanning/presets_list.md b/docs/scanning/presets_list.md new file mode 100644 index 000000000..3f82b118d --- /dev/null +++ b/docs/scanning/presets_list.md @@ -0,0 +1,338 @@ +Below is a list of every default BBOT preset, including its YAML. + + +## **cloud-enum** + +Enumerate cloud resources such as storage buckets, etc. + +??? note "`cloud-enum.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/cloud-enum.yml" + description: Enumerate cloud resources such as storage buckets, etc. + + include: + - subdomain-enum + + flags: + - cloud-enum + ``` + + + +Modules: [52]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `massdns`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman`, `rapiddns`, `riddler`, `securitytrails`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `threatminer`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") + +## **code-enum** + +Enumerate Git repositories, Docker images, etc. + +??? note "`code-enum.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/code-enum.yml" + description: Enumerate Git repositories, Docker images, etc. + + flags: + - code-enum + ``` + + + +Modules: [6]("`dockerhub`, `github_codesearch`, `github_org`, `gitlab`, `httpx`, `social`") + +## **dirbust-heavy** + +Recursive web directory brute-force (aggressive) + +??? note "`dirbust-heavy.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/dirbust-heavy.yml" + description: Recursive web directory brute-force (aggressive) + + include: + - spider + + flags: + - iis-shortnames + + modules: + - ffuf + - wayback + + config: + modules: + iis_shortnames: + # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames + detect_only: False + ffuf: + depth: 3 + lines: 5000 + extensions: + - php + - asp + - aspx + - ashx + - asmx + - jsp + - jspx + - cfm + - zip + - conf + - config + - xml + - json + - yml + - yaml + # emit URLs from wayback + wayback: + urls: True + ``` + +Category: web_advanced + +Modules: [5]("`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`, `wayback`") + +## **dirbust-light** + +Basic web directory brute-force (surface-level directories only) + +??? note "`dirbust-light.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/dirbust-light.yml" + description: Basic web directory brute-force (surface-level directories only) + + flags: + - iis-shortnames + + modules: + - ffuf + + config: + modules: + iis_shortnames: + # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames + detect_only: False + ffuf: + # wordlist size = 1000 + lines: 1000 + ``` + +Category: web_advanced + +Modules: [4]("`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`") + +## **email-enum** + +Enumerate email addresses from APIs, web crawling, etc. + +??? note "`email-enum.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/email-enum.yml" + description: Enumerate email addresses from APIs, web crawling, etc. + + flags: + - email-enum + + output_modules: + - emails + + config: + modules: + stdout: + format: text + # only output EMAIL_ADDRESSes to the console + event_types: + - EMAIL_ADDRESS + # only show in-scope emails + in_scope_only: True + # display the raw emails, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False + ``` + + + +Modules: [6]("`dehashed`, `emailformat`, `hunterio`, `pgp`, `skymem`, `sslcert`") + +## **kitchen-sink** + +Everything everywhere all at once + +??? note "`kitchen-sink.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/kitchen-sink.yml" + description: Everything everywhere all at once + + include: + - subdomain-enum + - cloud-enum + - code-enum + - email-enum + - spider + - web-basic + - paramminer + - dirbust-light + ``` + + + +Modules: [71]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `badsecrets`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `dehashed`, `digitorus`, `dnscommonsrv`, `dnsdumpster`, `dockerhub`, `emailformat`, `ffuf_shortnames`, `ffuf`, `filedownload`, `fullhunt`, `git`, `github_codesearch`, `github_org`, `gitlab`, `hackertarget`, `httpx`, `hunterio`, `iis_shortnames`, `internetdb`, `ipneighbor`, `leakix`, `massdns`, `myssl`, `ntlm`, `oauth`, `otx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `passivetotal`, `pgp`, `postman`, `rapiddns`, `riddler`, `robots`, `secretsdb`, `securitytrails`, `shodan_dns`, `sitedossier`, `skymem`, `social`, `sslcert`, `subdomaincenter`, `threatminer`, `urlscan`, `virustotal`, `wappalyzer`, `wayback`, `zoomeye`") + +## **paramminer** + +Discover new web parameters via brute-force + +??? note "`paramminer.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/paramminer.yml" + description: Discover new web parameters via brute-force + + flags: + - web-paramminer + + modules: + - httpx + + config: + web_spider_distance: 1 + web_spider_depth: 4 + ``` + +Category: web_advanced + +Modules: [4]("`httpx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`") + +## **secrets-enum** + + + +??? note "`secrets-enum.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/secrets-enum.yml" + description: + ``` + + + +Modules: [0]("") + +## **spider** + +Recursive web spider + +??? note "`spider.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/spider.yml" + description: Recursive web spider + + modules: + - httpx + + config: + # how many links to follow in a row + web_spider_distance: 2 + # don't follow links whose directory depth is higher than 4 + web_spider_depth: 4 + # maximum number of links to follow per page + web_spider_links_per_page: 25 + + modules: + stdout: + format: text + # only output URLs to the console + event_types: + - URL + # only show in-scope URLs + in_scope_only: True + # display the raw URLs, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False + ``` + + + +Modules: [1]("`httpx`") + +## **subdomain-enum** + +Enumerate subdomains via APIs, brute-force + +??? note "`subdomain-enum.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/subdomain-enum.yml" + description: Enumerate subdomains via APIs, brute-force + + flags: + - subdomain-enum + + output_modules: + - subdomains + + config: + modules: + stdout: + format: text + # only output DNS_NAMEs to the console + event_types: + - DNS_NAME + # only show in-scope subdomains + in_scope_only: True + # display the raw subdomains, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False + ``` + + + +Modules: [45]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `bevigil`, `binaryedge`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `massdns`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman`, `rapiddns`, `riddler`, `securitytrails`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `threatminer`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") + +## **web-basic** + +Quick web scan + +??? note "`web-basic.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/web-basic.yml" + description: Quick web scan + + flags: + - web-basic + ``` + + + +Modules: [17]("`azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_firebase`, `bucket_google`, `filedownload`, `git`, `httpx`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `secretsdb`, `sslcert`, `wappalyzer`") + +## **web-thorough** + +Aggressive web scan + +??? note "`web-thorough.yml`" + ```yaml title="~/Downloads/code/bbot/bbot/presets/web-thorough.yml" + description: Aggressive web scan + + include: + - web-basic + + flags: + - web-thorough + ``` + + + +Modules: [30]("`ajaxpro`, `azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_firebase`, `bucket_google`, `bypass403`, `dastardly`, `dotnetnuke`, `ffuf_shortnames`, `filedownload`, `generic_ssrf`, `git`, `host_header`, `httpx`, `hunt`, `iis_shortnames`, `nmap`, `ntlm`, `oauth`, `robots`, `secretsdb`, `smuggler`, `sslcert`, `telerik`, `url_manipulation`, `wappalyzer`") + + +## Table of Default Presets + +Here is a the same data, but in a table: + + +| Preset | Category | Description | # Modules | Modules | +|----------------|--------------|------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 52 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| code-enum | | Enumerate Git repositories, Docker images, etc. | 6 | dockerhub, github_codesearch, github_org, gitlab, httpx, social | +| dirbust-heavy | web_advanced | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | +| dirbust-light | web_advanced | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | +| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 6 | dehashed, emailformat, hunterio, pgp, skymem, sslcert | +| kitchen-sink | | Everything everywhere all at once | 71 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, gitlab, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, massdns, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wappalyzer, wayback, zoomeye | +| paramminer | web_advanced | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | +| secrets-enum | | | 0 | | +| spider | | Recursive web spider | 1 | httpx | +| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 45 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | +| web-basic | | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | +| web-thorough | | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | + diff --git a/mkdocs.yml b/mkdocs.yml index 62a1b4e95..a2d0c68c7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,7 +21,9 @@ nav: - Comparison to Other Tools: comparison.md - Scanning: - Scanning Overview: scanning/index.md - - Presets: scanning/presets.md + - Presets: + - Overview: scanning/presets.md + - List of Presets: scanning/presets_list.md - Events: scanning/events.md - Output: scanning/output.md - Tips and Tricks: scanning/tips_and_tricks.md @@ -55,6 +57,7 @@ theme: favicon: favicon.png features: - content.code.copy + - content.tooltips - navigation.tabs - navigation.sections - navigation.expand From fd467c698ca8c7f4ffc8e06218c82a6138831087 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 13:52:31 -0400 Subject: [PATCH 057/171] preset symlinking --- README.md | 12 ++--- bbot/cli.py | 2 + bbot/core/config/files.py | 38 ++----------- bbot/core/helpers/misc.py | 72 ------------------------- bbot/scanner/preset/preset.py | 77 ++++++++++++++++++++------- bbot/scripts/docs.py | 4 +- bbot/test/test_step_1/test_helpers.py | 67 ----------------------- docs/index.md | 12 ++--- docs/scanning/advanced.md | 12 ++--- docs/scanning/presets.md | 14 +++-- docs/scanning/presets_list.md | 24 ++++----- 11 files changed, 107 insertions(+), 227 deletions(-) diff --git a/README.md b/README.md index 7b6922b35..bae44b34b 100644 --- a/README.md +++ b/README.md @@ -73,42 +73,42 @@ Scan output, logs, etc. are saved to `~/.bbot`. For more detailed examples and e ```bash # Perform a full subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -f subdomain-enum +bbot -t evilcorp.com -p subdomain-enum ``` **Subdomains (passive only):** ```bash # Perform a passive-only subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -f subdomain-enum -rf passive +bbot -t evilcorp.com -p subdomain-enum -rf passive ``` **Subdomains + port scan + web screenshots:** ```bash # Port-scan every subdomain, screenshot every webpage, output to current directory -bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o . +bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o . ``` **Subdomains + basic web scan:** ```bash # A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules -bbot -t evilcorp.com -f subdomain-enum web-basic +bbot -t evilcorp.com -p subdomain-enum web-basic ``` **Web spider:** ```bash # Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc. -bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2 +bbot -t www.evilcorp.com -p spider -c web_spider_distance=2 web_spider_depth=2 ``` **Everything everywhere all at once:** ```bash # Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei -bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly +bbot -t evilcorp.com -p kitchen-sink ``` diff --git a/bbot/cli.py b/bbot/cli.py index 6cd2beb97..7d6c4ce26 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -31,6 +31,8 @@ async def _main(): # start by creating a default scan preset preset = Preset(_log=True, name="bbot_cli_main") + # populate preset symlinks + preset.all_presets # parse command line arguments and merge into preset try: preset.parse_args() diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index ec0cfb914..3313b5bab 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -2,9 +2,9 @@ from pathlib import Path from omegaconf import OmegaConf +from ..helpers.misc import mkdir from ..errors import ConfigLoadError from ..helpers.logger import log_to_stderr -from ..helpers.misc import mkdir, clean_dict, filter_dict bbot_code_dir = Path(__file__).parent.parent.parent @@ -15,17 +15,13 @@ class BBOTConfigFiles: config_dir = (Path.home() / ".config" / "bbot").resolve() defaults_filename = (bbot_code_dir / "defaults.yml").resolve() config_filename = (config_dir / "bbot.yml").resolve() - secrets_filename = (config_dir / "secrets.yml").resolve() def __init__(self, core): self.core = core - def ensure_config_files(self): + def ensure_config_file(self): mkdir(self.config_dir) - secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] - exclude_keys = ["modules"] - comment_notice = ( "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" + "# Please be sure to uncomment when inserting API keys, etc.\n" @@ -34,34 +30,11 @@ def ensure_config_files(self): # ensure bbot.yml if not self.config_filename.exists(): log_to_stderr(f"Creating BBOT config at {self.config_filename}") - no_secrets_config = OmegaConf.to_object(self.core.default_config) - no_secrets_config = clean_dict( - no_secrets_config, - *secrets_strings, - fuzzy=True, - exclude_keys=exclude_keys, - ) - yaml = OmegaConf.to_yaml(no_secrets_config) + yaml = OmegaConf.to_yaml(self.core.default_config) yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) with open(str(self.config_filename), "w") as f: f.write(yaml) - # ensure secrets.yml - if not self.secrets_filename.exists(): - log_to_stderr(f"Creating BBOT secrets at {self.secrets_filename}") - secrets_only_config = OmegaConf.to_object(self.core.default_config) - secrets_only_config = filter_dict( - secrets_only_config, - *secrets_strings, - fuzzy=True, - exclude_keys=exclude_keys, - ) - yaml = OmegaConf.to_yaml(secrets_only_config) - yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) - with open(str(self.secrets_filename), "w") as f: - f.write(yaml) - self.secrets_filename.chmod(0o600) - def _get_config(self, filename, name="config"): filename = Path(filename).resolve() try: @@ -76,10 +49,7 @@ def _get_config(self, filename, name="config"): return OmegaConf.create() def get_custom_config(self): - return OmegaConf.merge( - self._get_config(self.config_filename, name="config"), - self._get_config(self.secrets_filename, name="secrets"), - ) + return self._get_config(self.config_filename, name="config") def get_default_config(self): return self._get_config(self.defaults_filename, name="defaults") diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 1ae26a425..c229b26a1 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1471,78 +1471,6 @@ def search_dict_values(d, *regexes): yield from search_dict_values(v, *regexes) -def filter_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): - """ - Recursively filter a dictionary based on key names. - - Args: - d (dict): The input dictionary. - *key_names: Names of keys to filter for. - fuzzy (bool): Whether to perform fuzzy matching on keys. - exclude_keys (list, None): List of keys to be excluded from the final dict. - _prev_key (str, None): For internal recursive use; the previous key in the hierarchy. - - Returns: - dict: A dictionary containing only the keys specified in key_names. - - Examples: - >>> filter_dict({"key1": "test", "key2": "asdf"}, "key2") - {"key2": "asdf"} - >>> filter_dict({"key1": "test", "key2": {"key3": "asdf"}}, "key1", "key3", exclude_keys="key2") - {'key1': 'test'} - """ - import copy - - if exclude_keys is None: - exclude_keys = [] - if isinstance(exclude_keys, str): - exclude_keys = [exclude_keys] - ret = {} - if isinstance(d, dict): - for key in d: - if key in key_names or (fuzzy and any(k in key for k in key_names)): - if not any(k in exclude_keys for k in [key, _prev_key]): - ret[key] = copy.deepcopy(d[key]) - elif isinstance(d[key], list) or isinstance(d[key], dict): - child = filter_dict(d[key], *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys) - if child: - ret[key] = child - return ret - - -def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): - """ - Recursively clean unwanted keys from a dictionary. - Useful for removing secrets from a config. - - Args: - d (dict): The input dictionary. - *key_names: Names of keys to remove. - fuzzy (bool): Whether to perform fuzzy matching on keys. - exclude_keys (list, None): List of keys to be excluded from removal. - _prev_key (str, None): For internal recursive use; the previous key in the hierarchy. - - Returns: - dict: A dictionary cleaned of the keys specified in key_names. - - """ - import copy - - if exclude_keys is None: - exclude_keys = [] - if isinstance(exclude_keys, str): - exclude_keys = [exclude_keys] - d = copy.deepcopy(d) - if isinstance(d, dict): - for key, val in list(d.items()): - if key in key_names or (fuzzy and any(k in key for k in key_names)): - if _prev_key not in exclude_keys: - d.pop(key) - else: - d[key] = clean_dict(val, *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys) - return d - - def grouper(iterable, n): """ Grouper groups an iterable into chunks of a given size. diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index b72cd8652..6eca7ef2b 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -12,12 +12,16 @@ from bbot.core import CORE from bbot.core.errors import * from bbot.core.event.base import make_event -from bbot.core.helpers.misc import make_table +from bbot.core.helpers.misc import make_table, mkdir log = logging.getLogger("bbot.presets") +# cache default presets to prevent having to reload from disk +DEFAULT_PRESETS = None + + class Preset: def __init__( @@ -146,6 +150,10 @@ def __init__( def bbot_home(self): return Path(self.config.get("home", "~/.bbot")).expanduser().resolve() + @property + def preset_dir(self): + return self.bbot_home / "presets" + def merge(self, other): # config self.core.merge_custom(other.core.custom_config) @@ -684,31 +692,64 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) - @classmethod - def all_presets(cls): - presets = dict() - for ext in ("yml", "yaml"): - for preset_path in PRESET_PATH: - for yaml_file in preset_path.rglob(f"**/*.{ext}"): - try: - loaded_preset = cls.from_yaml_file(yaml_file, _log=True) - category = str(yaml_file.relative_to(preset_path).parent) + def all_presets(self): + preset_dir = self.preset_dir + home_dir = Path.home() + + # first, add local preset dir to PRESET_PATH + PRESET_PATH.add_path(self.preset_dir) + + # ensure local preset directory exists + mkdir(preset_dir) + + global DEFAULT_PRESETS + if DEFAULT_PRESETS is None: + presets = dict() + for ext in ("yml", "yaml"): + for preset_path in PRESET_PATH: + # for every yaml file + for original_filename in preset_path.rglob(f"**/*.{ext}"): + # not including symlinks + if original_filename.is_symlink(): + continue + + # try to load it as a preset + try: + loaded_preset = self.from_yaml_file(original_filename, _log=True) + except Exception as e: + log.warning(f'Failed to load preset at "{original_filename}": {e}') + log.trace(traceback.format_exc()) + continue + + # category is the parent folder(s), if any + category = str(original_filename.relative_to(preset_path).parent) if category == ".": category = "" - presets[yaml_file] = (loaded_preset, category) - except Exception as e: - log.warning(f'Failed to load preset at "{yaml_file}": {e}') - log.trace(traceback.format_exc()) - continue - # sort by name - return dict(sorted(presets.items(), key=lambda x: x[-1][0].name)) + + local_preset = original_filename + # populate symlinks in local preset dir + if not original_filename.is_relative_to(preset_dir): + relative_preset = original_filename.relative_to(preset_path) + local_preset = preset_dir / relative_preset + mkdir(local_preset.parent, check_writable=False) + if not local_preset.exists(): + local_preset.symlink_to(original_filename) + + if local_preset.is_relative_to(home_dir): + local_preset = Path("~") / local_preset.relative_to(home_dir) + + presets[local_preset] = (loaded_preset, category, preset_path, original_filename) + + # sort by name + DEFAULT_PRESETS = dict(sorted(presets.items(), key=lambda x: x[-1][0].name)) + return DEFAULT_PRESETS def presets_table(self, include_modules=True): table = [] header = ["Preset", "Category", "Description", "# Modules"] if include_modules: header.append("Modules") - for yaml_file, (loaded_preset, category) in self.all_presets().items(): + for yaml_file, (loaded_preset, category, preset_path, original_file) in self.all_presets().items(): num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] if include_modules: diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index 84546f936..f81628cb7 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -126,7 +126,7 @@ def update_individual_module_options(): update_md_files("BBOT PRESETS", bbot_presets_table) # BBOT subdomain enum preset - for yaml_file, (loaded_preset, category) in DEFAULT_PRESET.all_presets().items(): + for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items(): if loaded_preset.name == "subdomain-enum": subdomain_enum_preset = f"""```yaml title="{yaml_file.name}" {loaded_preset._yaml_str} @@ -135,7 +135,7 @@ def update_individual_module_options(): break content = [] - for yaml_file, (loaded_preset, category) in DEFAULT_PRESET.all_presets().items(): + for yaml_file, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items(): yaml_str = loaded_preset._yaml_str indent = " " * 4 yaml_str = f"\n{indent}".join(yaml_str.splitlines()) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index c8045e595..4780eb522 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -254,73 +254,6 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https "https://www.evilcorp.com/fdsa", } - filtered_dict = helpers.filter_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, "api_key" - ) - assert "api_key" in filtered_dict["modules"]["c99"] - assert "filterme" not in filtered_dict["modules"]["c99"] - assert "ipneighbor" not in filtered_dict["modules"] - - filtered_dict2 = helpers.filter_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, "c99" - ) - assert "api_key" in filtered_dict2["modules"]["c99"] - assert "filterme" in filtered_dict2["modules"]["c99"] - assert "ipneighbor" not in filtered_dict2["modules"] - - filtered_dict3 = helpers.filter_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, - "key", - fuzzy=True, - ) - assert "api_key" in filtered_dict3["modules"]["c99"] - assert "filterme" not in filtered_dict3["modules"]["c99"] - assert "ipneighbor" not in filtered_dict3["modules"] - - filtered_dict4 = helpers.filter_dict( - {"modules": {"secrets_db": {"api_key": "1234"}, "ipneighbor": {"secret": "test", "asdf": "1234"}}}, - "secret", - fuzzy=True, - exclude_keys="modules", - ) - assert not "secrets_db" in filtered_dict4["modules"] - assert "ipneighbor" in filtered_dict4["modules"] - assert "secret" in filtered_dict4["modules"]["ipneighbor"] - assert "asdf" not in filtered_dict4["modules"]["ipneighbor"] - - cleaned_dict = helpers.clean_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, "api_key" - ) - assert "api_key" not in cleaned_dict["modules"]["c99"] - assert "filterme" in cleaned_dict["modules"]["c99"] - assert "ipneighbor" in cleaned_dict["modules"] - - cleaned_dict2 = helpers.clean_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, "c99" - ) - assert "c99" not in cleaned_dict2["modules"] - assert "ipneighbor" in cleaned_dict2["modules"] - - cleaned_dict3 = helpers.clean_dict( - {"modules": {"c99": {"api_key": "1234", "filterme": "asdf"}, "ipneighbor": {"test": "test"}}}, - "key", - fuzzy=True, - ) - assert "api_key" not in cleaned_dict3["modules"]["c99"] - assert "filterme" in cleaned_dict3["modules"]["c99"] - assert "ipneighbor" in cleaned_dict3["modules"] - - cleaned_dict4 = helpers.clean_dict( - {"modules": {"secrets_db": {"api_key": "1234"}, "ipneighbor": {"secret": "test", "asdf": "1234"}}}, - "secret", - fuzzy=True, - exclude_keys="modules", - ) - assert "secrets_db" in cleaned_dict4["modules"] - assert "ipneighbor" in cleaned_dict4["modules"] - assert "secret" not in cleaned_dict4["modules"]["ipneighbor"] - assert "asdf" in cleaned_dict4["modules"]["ipneighbor"] - replaced = helpers.search_format_dict( {"asdf": [{"wat": {"here": "#{replaceme}!"}}, {500: True}]}, replaceme="asdf" ) diff --git a/docs/index.md b/docs/index.md index ae590beef..f2f474c45 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,42 +55,42 @@ Below are some examples of common scans. ```bash # Perform a full subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -f subdomain-enum +bbot -t evilcorp.com -p subdomain-enum ``` **Subdomains (passive only):** ```bash # Perform a passive-only subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -f subdomain-enum -rf passive +bbot -t evilcorp.com -p subdomain-enum -rf passive ``` **Subdomains + port scan + web screenshots:** ```bash # Port-scan every subdomain, screenshot every webpage, output to current directory -bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o . +bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o . ``` **Subdomains + basic web scan:** ```bash # A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules -bbot -t evilcorp.com -f subdomain-enum web-basic +bbot -t evilcorp.com -p subdomain-enum web-basic ``` **Web spider:** ```bash # Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc. -bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2 +bbot -t www.evilcorp.com -p spider -c web_spider_distance=2 web_spider_depth=2 ``` **Everything everywhere all at once:** ```bash # Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei -bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly +bbot -t evilcorp.com -p kitchen-sink ``` diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 8de312b86..12474f0f1 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -107,22 +107,22 @@ Misc: EXAMPLES Subdomains: - bbot -t evilcorp.com -f subdomain-enum + bbot -t evilcorp.com -p subdomain-enum Subdomains (passive only): - bbot -t evilcorp.com -f subdomain-enum -rf passive + bbot -t evilcorp.com -p subdomain-enum -rf passive Subdomains + port scan + web screenshots: - bbot -t evilcorp.com -f subdomain-enum -m nmap gowitness -n my_scan -o . + bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o . Subdomains + basic web scan: - bbot -t evilcorp.com -f subdomain-enum web-basic + bbot -t evilcorp.com -p subdomain-enum web-basic Web spider: - bbot -t www.evilcorp.com -m httpx robots badsecrets secretsdb -c web_spider_distance=2 web_spider_depth=2 + bbot -t www.evilcorp.com -p spider -c web_spider_distance=2 web_spider_depth=2 Everything everywhere all at once: - bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap gowitness nuclei --allow-deadly + bbot -t evilcorp.com -p kitchen-sink List modules: bbot -l diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index 854fa8064..a2ed4ad2d 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -34,7 +34,16 @@ config: ## How to use Presets (`-p`) -BBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`. You can use them like this: +BBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`. + +To list them, you can do: + +```bash +# list available presets +bbot -lp +``` + +Enable them with `-p`: ```bash # do a subdomain enumeration @@ -48,9 +57,6 @@ bbot -t evilcorp.com -p subdomain-enum -rf passive # preset + manual config override bbot -t www.evilcorp.com -p spider -c web_spider_distance=10 - -# list available presets -bbot -lp ``` You can also build on the default presets, or create your own. Here's an example of a custom preset that builds on `subdomain-enum`: diff --git a/docs/scanning/presets_list.md b/docs/scanning/presets_list.md index 3f82b118d..692e0ce4a 100644 --- a/docs/scanning/presets_list.md +++ b/docs/scanning/presets_list.md @@ -6,7 +6,7 @@ Below is a list of every default BBOT preset, including its YAML. Enumerate cloud resources such as storage buckets, etc. ??? note "`cloud-enum.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/cloud-enum.yml" + ```yaml title="~/.bbot/presets/cloud-enum.yml" description: Enumerate cloud resources such as storage buckets, etc. include: @@ -25,7 +25,7 @@ Modules: [52]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, Enumerate Git repositories, Docker images, etc. ??? note "`code-enum.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/code-enum.yml" + ```yaml title="~/.bbot/presets/code-enum.yml" description: Enumerate Git repositories, Docker images, etc. flags: @@ -41,7 +41,7 @@ Modules: [6]("`dockerhub`, `github_codesearch`, `github_org`, `gitlab`, `httpx`, Recursive web directory brute-force (aggressive) ??? note "`dirbust-heavy.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/dirbust-heavy.yml" + ```yaml title="~/.bbot/presets/web_advanced/dirbust-heavy.yml" description: Recursive web directory brute-force (aggressive) include: @@ -92,7 +92,7 @@ Modules: [5]("`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`, `wayback`") Basic web directory brute-force (surface-level directories only) ??? note "`dirbust-light.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/dirbust-light.yml" + ```yaml title="~/.bbot/presets/web_advanced/dirbust-light.yml" description: Basic web directory brute-force (surface-level directories only) flags: @@ -120,7 +120,7 @@ Modules: [4]("`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`") Enumerate email addresses from APIs, web crawling, etc. ??? note "`email-enum.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/email-enum.yml" + ```yaml title="~/.bbot/presets/email-enum.yml" description: Enumerate email addresses from APIs, web crawling, etc. flags: @@ -154,7 +154,7 @@ Modules: [6]("`dehashed`, `emailformat`, `hunterio`, `pgp`, `skymem`, `sslcert`" Everything everywhere all at once ??? note "`kitchen-sink.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/kitchen-sink.yml" + ```yaml title="~/.bbot/presets/kitchen-sink.yml" description: Everything everywhere all at once include: @@ -177,7 +177,7 @@ Modules: [71]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, Discover new web parameters via brute-force ??? note "`paramminer.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/web_advanced/paramminer.yml" + ```yaml title="~/.bbot/presets/web_advanced/paramminer.yml" description: Discover new web parameters via brute-force flags: @@ -200,7 +200,7 @@ Modules: [4]("`httpx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer ??? note "`secrets-enum.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/secrets-enum.yml" + ```yaml title="~/.bbot/presets/secrets-enum.yml" description: ``` @@ -213,7 +213,7 @@ Modules: [0]("") Recursive web spider ??? note "`spider.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/spider.yml" + ```yaml title="~/.bbot/presets/spider.yml" description: Recursive web spider modules: @@ -251,7 +251,7 @@ Modules: [1]("`httpx`") Enumerate subdomains via APIs, brute-force ??? note "`subdomain-enum.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/subdomain-enum.yml" + ```yaml title="~/.bbot/presets/subdomain-enum.yml" description: Enumerate subdomains via APIs, brute-force flags: @@ -285,7 +285,7 @@ Modules: [45]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, Quick web scan ??? note "`web-basic.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/web-basic.yml" + ```yaml title="~/.bbot/presets/web-basic.yml" description: Quick web scan flags: @@ -301,7 +301,7 @@ Modules: [17]("`azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_a Aggressive web scan ??? note "`web-thorough.yml`" - ```yaml title="~/Downloads/code/bbot/bbot/presets/web-thorough.yml" + ```yaml title="~/.bbot/presets/web-thorough.yml" description: Aggressive web scan include: From 91da2cf9e5400d5533cdac323fce1560f8d1cc05 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 14:20:09 -0400 Subject: [PATCH 058/171] ensure presets have descriptions --- bbot/modules/azure_realm.py | 2 +- bbot/modules/badsecrets.py | 2 +- bbot/modules/bucket_amazon.py | 2 +- bbot/modules/bucket_azure.py | 2 +- bbot/modules/bucket_firebase.py | 2 +- bbot/modules/bucket_google.py | 2 +- bbot/modules/filedownload.py | 2 +- bbot/modules/git.py | 2 +- bbot/modules/httpx.py | 2 +- bbot/modules/iis_shortnames.py | 2 +- bbot/modules/ntlm.py | 2 +- bbot/modules/oauth.py | 2 +- bbot/modules/postman.py | 2 +- bbot/modules/robots.py | 2 +- bbot/modules/secretsdb.py | 2 +- bbot/modules/sslcert.py | 2 +- bbot/modules/templates/bucket.py | 2 +- bbot/modules/trufflehog.py | 2 +- bbot/modules/wappalyzer.py | 2 +- bbot/presets/secrets-enum.yml | 1 - bbot/scanner/preset/preset.py | 4 +++- bbot/test/test_step_1/test_modules_basic.py | 3 +++ bbot/test/test_step_1/test_presets.py | 9 +++++++++ 23 files changed, 34 insertions(+), 21 deletions(-) delete mode 100644 bbot/presets/secrets-enum.yml diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py index 33772921e..a3d6ad6ba 100644 --- a/bbot/modules/azure_realm.py +++ b/bbot/modules/azure_realm.py @@ -4,7 +4,7 @@ class azure_realm(BaseModule): watched_events = ["DNS_NAME"] produced_events = ["URL_UNVERIFIED"] - flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "web-thorough", "passive", "safe"] + flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "passive", "safe"] meta = {"description": 'Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm'} async def setup(self): diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 7fde4a8e3..5626314fe 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -8,7 +8,7 @@ class badsecrets(BaseModule): watched_events = ["HTTP_RESPONSE"] produced_events = ["FINDING", "VULNERABILITY", "TECHNOLOGY"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = {"description": "Library for detecting known or weak secrets across many web frameworks"} deps_pip = ["badsecrets~=0.4.490"] diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py index ddb6d05f3..3e17b186a 100644 --- a/bbot/modules/bucket_amazon.py +++ b/bbot/modules/bucket_amazon.py @@ -4,7 +4,7 @@ class bucket_amazon(bucket_template): watched_events = ["DNS_NAME", "STORAGE_BUCKET"] produced_events = ["STORAGE_BUCKET", "FINDING"] - flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"] + flags = ["active", "safe", "cloud-enum", "web-basic"] meta = {"description": "Check for S3 buckets related to target"} options = {"permutations": False} options_desc = { diff --git a/bbot/modules/bucket_azure.py b/bbot/modules/bucket_azure.py index 81e441b5f..6c828afed 100644 --- a/bbot/modules/bucket_azure.py +++ b/bbot/modules/bucket_azure.py @@ -4,7 +4,7 @@ class bucket_azure(bucket_template): watched_events = ["DNS_NAME", "STORAGE_BUCKET"] produced_events = ["STORAGE_BUCKET", "FINDING"] - flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"] + flags = ["active", "safe", "cloud-enum", "web-basic"] meta = {"description": "Check for Azure storage blobs related to target"} options = {"permutations": False} options_desc = { diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index 9c883bc9c..01b1fc213 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -4,7 +4,7 @@ class bucket_firebase(bucket_template): watched_events = ["DNS_NAME", "STORAGE_BUCKET"] produced_events = ["STORAGE_BUCKET", "FINDING"] - flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"] + flags = ["active", "safe", "cloud-enum", "web-basic"] meta = {"description": "Check for open Firebase databases related to target"} options = {"permutations": False} options_desc = { diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py index 0bd22f0f1..9e63ddc8b 100644 --- a/bbot/modules/bucket_google.py +++ b/bbot/modules/bucket_google.py @@ -8,7 +8,7 @@ class bucket_google(bucket_template): watched_events = ["DNS_NAME", "STORAGE_BUCKET"] produced_events = ["STORAGE_BUCKET", "FINDING"] - flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"] + flags = ["active", "safe", "cloud-enum", "web-basic"] meta = {"description": "Check for Google object storage related to target"} options = {"permutations": False} options_desc = { diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index 5cf190d1f..05db2b9df 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -14,7 +14,7 @@ class filedownload(BaseModule): watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"] produced_events = [] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = {"description": "Download common filetypes such as PDF, DOCX, PPTX, etc."} options = { "extensions": [ diff --git a/bbot/modules/git.py b/bbot/modules/git.py index 0b19f7e6a..5ffb91331 100644 --- a/bbot/modules/git.py +++ b/bbot/modules/git.py @@ -6,7 +6,7 @@ class git(BaseModule): watched_events = ["URL"] produced_events = ["FINDING"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic", "code-enum"] meta = {"description": "Check for exposed .git repositories"} in_scope_only = True diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 2e5dc0ffc..30cc3fba3 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -10,7 +10,7 @@ class httpx(BaseModule): watched_events = ["OPEN_TCP_PORT", "URL_UNVERIFIED", "URL"] produced_events = ["URL", "HTTP_RESPONSE"] - flags = ["active", "safe", "web-basic", "web-thorough", "social-enum", "subdomain-enum", "cloud-enum"] + flags = ["active", "safe", "web-basic", "social-enum", "subdomain-enum", "cloud-enum"] meta = {"description": "Visit webpages. Many other modules rely on httpx"} options = { diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 7d558a23a..94d325df8 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -16,7 +16,7 @@ class IISShortnamesError(Exception): class iis_shortnames(BaseModule): watched_events = ["URL"] produced_events = ["URL_HINT"] - flags = ["active", "safe", "web-basic", "web-thorough", "iis-shortnames"] + flags = ["active", "safe", "web-basic", "iis-shortnames"] meta = {"description": "Check for IIS shortname vulnerability"} options = {"detect_only": True, "max_node_count": 50} options_desc = { diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index c69beb941..8f07b4728 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -68,7 +68,7 @@ class ntlm(BaseModule): watched_events = ["URL", "HTTP_RESPONSE"] produced_events = ["FINDING", "DNS_NAME"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = {"description": "Watch for HTTP endpoints that support NTLM authentication"} options = {"try_all": False} options_desc = {"try_all": "Try every NTLM endpoint"} diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 13a483aad..7e376000e 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -6,7 +6,7 @@ class OAUTH(BaseModule): watched_events = ["DNS_NAME", "URL_UNVERIFIED"] produced_events = ["DNS_NAME"] - flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "web-thorough", "active", "safe"] + flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "active", "safe"] meta = {"description": "Enumerate OAUTH and OpenID Connect services"} options = {"try_all": False} options_desc = {"try_all": "Check for OAUTH/IODC on every subdomain and URL."} diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 5a63f824e..348d0dc28 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -4,7 +4,7 @@ class postman(subdomain_enum): watched_events = ["DNS_NAME"] produced_events = ["URL_UNVERIFIED"] - flags = ["passive", "subdomain-enum", "safe"] + flags = ["passive", "subdomain-enum", "safe", "code-enum"] meta = {"description": "Query Postman's API for related workspaces, collections, requests"} base_url = "https://www.postman.com/_api" diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index 717900bee..d801a755e 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -4,7 +4,7 @@ class robots(BaseModule): watched_events = ["URL"] produced_events = ["URL_UNVERIFIED"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = {"description": "Look for and parse robots.txt"} options = {"include_sitemap": False, "include_allow": True, "include_disallow": True} diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py index 3fc8ad539..d9462ae19 100644 --- a/bbot/modules/secretsdb.py +++ b/bbot/modules/secretsdb.py @@ -7,7 +7,7 @@ class secretsdb(BaseModule): watched_events = ["HTTP_RESPONSE"] produced_events = ["FINDING"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = {"description": "Detect common secrets with secrets-patterns-db"} options = { "min_confidence": 99, diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index caca5feca..edc4a484f 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -10,7 +10,7 @@ class sslcert(BaseModule): watched_events = ["OPEN_TCP_PORT"] produced_events = ["DNS_NAME", "EMAIL_ADDRESS"] - flags = ["affiliates", "subdomain-enum", "email-enum", "active", "safe", "web-basic", "web-thorough"] + flags = ["affiliates", "subdomain-enum", "email-enum", "active", "safe", "web-basic"] meta = { "description": "Visit open ports and retrieve SSL certificates", } diff --git a/bbot/modules/templates/bucket.py b/bbot/modules/templates/bucket.py index f9681385b..095da6b70 100644 --- a/bbot/modules/templates/bucket.py +++ b/bbot/modules/templates/bucket.py @@ -4,7 +4,7 @@ class bucket_template(BaseModule): watched_events = ["DNS_NAME", "STORAGE_BUCKET"] produced_events = ["STORAGE_BUCKET", "FINDING"] - flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"] + flags = ["active", "safe", "cloud-enum", "web-basic"] options = {"permutations": False} options_desc = { "permutations": "Whether to try permutations", diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 0bf27b413..d9c27b239 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -5,7 +5,7 @@ class trufflehog(BaseModule): watched_events = ["FILESYSTEM"] produced_events = ["FINDING", "VULNERABILITY"] - flags = ["passive", "safe"] + flags = ["passive", "safe", "code-enum"] meta = {"description": "TruffleHog is a tool for finding credentials"} options = { diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index c87274a29..00e18f429 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -13,7 +13,7 @@ class wappalyzer(BaseModule): watched_events = ["HTTP_RESPONSE"] produced_events = ["TECHNOLOGY"] - flags = ["active", "safe", "web-basic", "web-thorough"] + flags = ["active", "safe", "web-basic"] meta = { "description": "Extract technologies from web responses", } diff --git a/bbot/presets/secrets-enum.yml b/bbot/presets/secrets-enum.yml deleted file mode 100644 index b97271f79..000000000 --- a/bbot/presets/secrets-enum.yml +++ /dev/null @@ -1 +0,0 @@ -description: diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 6eca7ef2b..9b0f6f302 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -331,7 +331,7 @@ def add_module(self, module_name, module_type="scan"): return if module_name not in self.modules: - self.log_verbose(f'Adding module "{module_name}"') + self.log_debug(f'Adding module "{module_name}"') self.modules.add(module_name) for module_dep in preloaded.get("deps", {}).get("modules", []): if module_dep not in self.modules: @@ -692,6 +692,7 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) + @property def all_presets(self): preset_dir = self.preset_dir home_dir = Path.home() @@ -735,6 +736,7 @@ def all_presets(self): if not local_preset.exists(): local_preset.symlink_to(original_filename) + # collapse home directory into "~" if local_preset.is_relative_to(home_dir): local_preset = Path("~") / local_preset.relative_to(home_dir) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index b66f06bbf..95a70afc1 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -129,6 +129,9 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h assert ("safe" in flags and not "aggressive" in flags) or ( not "safe" in flags and "aggressive" in flags ), f'module "{module_name}" must have either "safe" or "aggressive" flag' + assert not ( + "web-basic" in flags and "web-thorough" in flags + ), f'module "{module_name}" should have either "web-basic" or "web-thorough" flags, not both' assert preloaded.get("meta", {}).get("description", ""), f"{module_name} must have a description" # attribute checks diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index b8f9f9978..e61c22d77 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -3,6 +3,15 @@ from bbot.scanner import Scanner, Preset +def test_preset_descriptions(): + # ensure very preset has a description + preset = Preset() + for yaml_file, (loaded_preset, category, preset_path, original_filename) in preset.all_presets.items(): + assert ( + loaded_preset.description + ), f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.' + + def test_core(): from bbot.core import CORE From b6e6c9f88e480f6e99e6e73393df595aff272f53 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 21:27:03 -0400 Subject: [PATCH 059/171] steady work on tests --- bbot/core/modules.py | 2 +- bbot/scanner/preset/preset.py | 19 +++++--- bbot/scanner/scanner.py | 1 - bbot/scripts/docs.py | 1 - bbot/test/bbot_fixtures.py | 48 ++++++++++--------- bbot/test/test.conf | 9 ---- bbot/test/test_step_1/test_cli.py | 11 ++--- bbot/test/test_step_1/test_command.py | 8 ++-- bbot/test/test_step_1/test_config.py | 15 +++++- bbot/test/test_step_1/test_depsinstaller.py | 4 +- .../test_manager_scope_accuracy.py | 20 ++++---- bbot/test/test_step_1/test_presets.py | 43 +++++++++++------ bbot/test/test_step_1/test_python_api.py | 14 +++--- bbot/test/test_step_2/module_tests/base.py | 2 +- .../module_tests/test_module_nmap.py | 4 +- docs/dev/index.md | 12 +---- 16 files changed, 114 insertions(+), 99 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index c6d050a57..db34a39c6 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -645,4 +645,4 @@ def filter_modules(self, modules=None, mod_type=None): return module_list -module_loader = ModuleLoader() +MODULE_LOADER = ModuleLoader() diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9b0f6f302..e23bd3524 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -214,6 +214,11 @@ def bake(self): os.environ.clear() os.environ.update(os_environ) + # disable internal modules if requested + for internal_module in baked_preset.internal_modules: + if baked_preset.config.get(internal_module, True) == False: + baked_preset.exclude_module(internal_module) + # evaluate conditions if baked_preset.conditions: from .conditions import ConditionEvaluator @@ -430,7 +435,7 @@ def add_excluded_modules(self, modules): def exclude_module(self, module): self.exclude_modules.add(module) - for module in list(self.scan_modules): + for module in list(self.modules): if module in self.exclude_modules: self.log_verbose(f'Removing module "{module}" because it\'s excluded') self.modules.remove(module) @@ -505,9 +510,9 @@ def helpers(self): def module_loader(self): self.environ if self._module_loader is None: - from bbot.core.modules import module_loader + from bbot.core.modules import MODULE_LOADER - self._module_loader = module_loader + self._module_loader = MODULE_LOADER return self._module_loader @@ -596,7 +601,7 @@ def include_preset(self, filename): preset_from_yaml = self.from_yaml_file(preset_filename, _exclude=self._preset_files_loaded) if preset_from_yaml is not False: self.merge(preset_from_yaml) - self._preset_files_loaded.add(preset_filename) + self._preset_files_loaded.add(preset_filename) @classmethod def from_yaml_file(cls, filename, _exclude=None, _log=False): @@ -605,9 +610,9 @@ def from_yaml_file(cls, filename, _exclude=None, _log=False): The file extension is optional. """ + filename = Path(filename).resolve() if _exclude is None: _exclude = set() - filename = Path(filename).resolve() if _exclude is not None and filename in _exclude: log.debug(f"Not loading {filename} because it was already loaded {_exclude}") return False @@ -717,6 +722,8 @@ def all_presets(self): # try to load it as a preset try: loaded_preset = self.from_yaml_file(original_filename, _log=True) + if loaded_preset is False: + continue except Exception as e: log.warning(f'Failed to load preset at "{original_filename}": {e}') log.trace(traceback.format_exc()) @@ -751,7 +758,7 @@ def presets_table(self, include_modules=True): header = ["Preset", "Category", "Description", "# Modules"] if include_modules: header.append("Modules") - for yaml_file, (loaded_preset, category, preset_path, original_file) in self.all_presets().items(): + for yaml_file, (loaded_preset, category, preset_path, original_file) in self.all_presets.items(): num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] if include_modules: diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 4cb0c8a8f..c1dd1bd14 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -920,7 +920,6 @@ def _log_handlers(self): return self.__log_handlers def _start_log_handlers(self): - # PRESET TODO: revisit scan logging # add log handlers for handler in self._log_handlers: self.core.logger.add_log_handler(handler) diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index f81628cb7..f8a5050a3 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -58,7 +58,6 @@ def find_replace_file(file, keyword, replace): f.write(new_content) -# PRESET TODO: revisit this def update_docs(): md_files = [p for p in bbot_code_dir.glob("**/*.md") if p.is_file()] diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 1703e0646..bd9dd68f1 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -1,3 +1,4 @@ +import os # noqa import dns import sys import pytest @@ -7,15 +8,40 @@ import tldextract import pytest_httpserver from pathlib import Path +from omegaconf import OmegaConf from werkzeug.wrappers import Request +from bbot.core import CORE +from bbot.scanner import Preset from bbot.core.helpers.misc import mkdir +log = logging.getLogger(f"bbot.test.fixtures") + + bbot_test_dir = Path("/tmp/.bbot_test") mkdir(bbot_test_dir) +# bbot config + +test_config = OmegaConf.load(Path(__file__).parent / "test.conf") +CORE.merge_custom(test_config) + +if test_config.get("debug", False): + logging.getLogger("bbot").setLevel(logging.DEBUG) +else: + # silence stdout + trace + root_logger = logging.getLogger() + for h in root_logger.handlers: + h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) + +default_preset = Preset() + +available_modules = list(default_preset.module_loader.configs(type="scan")) +available_output_modules = list(default_preset.module_loader.configs(type="output")) +available_internal_modules = list(default_preset.module_loader.configs(type="internal")) + class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): def match_data(self, request: Request) -> bool: @@ -32,15 +58,8 @@ def match_data(self, request: Request) -> bool: log = logging.getLogger("werkzeug") log.setLevel(logging.CRITICAL) -# silence stdout -root_logger = logging.getLogger() -for h in root_logger.handlers: - h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) - tldextract.extract("www.evilcorp.com") -log = logging.getLogger(f"bbot.test.fixtures") - @pytest.fixture def bbot_scanner(): @@ -203,21 +222,6 @@ def agent(monkeypatch, bbot_config): return test_agent -# bbot config -from bbot.scanner import Preset - -default_preset = Preset.from_yaml_file(Path(__file__).parent / "test.conf") -test_config = default_preset.config - -available_modules = list(default_preset.module_loader.configs(type="scan")) -available_output_modules = list(default_preset.module_loader.configs(type="output")) -available_internal_modules = list(default_preset.module_loader.configs(type="internal")) - - -if test_config.get("debug", False): - logging.getLogger("bbot").setLevel(logging.DEBUG) - - @pytest.fixture def bbot_config(): return test_config diff --git a/bbot/test/test.conf b/bbot/test/test.conf index ba8367461..cadcca687 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -4,9 +4,6 @@ modules: wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt ffuf: prefix_busting: true - ipneighbor: - test_option: ipneighbor -output_modules: http: url: http://127.0.0.1:11111 username: username @@ -17,18 +14,12 @@ output_modules: token: asdf neo4j: uri: bolt://127.0.0.1:11111 - python: - test_option: asdf -internal_modules: - speculate: - test_option: speculate http_proxy: http_headers: { "test": "header" } ssl_verify: false scope_search_distance: 0 scope_report_distance: 0 scope_dns_search_distance: 1 -plumbus: asdf dns_debug: false user_agent: "BBOT Test User-Agent" http_debug: false diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 309699d7f..bfe504268 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -1,6 +1,3 @@ -import os -import sys - from ..bbot_fixtures import * @@ -143,8 +140,8 @@ async def test_cli_args(monkeypatch, capsys): # --allow-deadly monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei", "--allow-deadly"]) - # result = await cli._main() - # assert result == True, "-m nuclei failed to run with --allow-deadly" + result = await cli._main() + assert result == True, "-m nuclei failed to run with --allow-deadly" # install all deps # monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"]) @@ -232,7 +229,7 @@ def test_cli_presets(monkeypatch, capsys): preset_dir = bbot_test_dir / "test_cli_presets" preset_dir.mkdir(exist_ok=True) - preset1_file = preset_dir / "preset1.conf" + preset1_file = preset_dir / "cli_preset1.conf" with open(preset1_file, "w") as f: f.write( """ @@ -241,7 +238,7 @@ def test_cli_presets(monkeypatch, capsys): """ ) - preset2_file = preset_dir / "preset2.yml" + preset2_file = preset_dir / "cli_preset2.yml" with open(preset2_file, "w") as f: f.write( """ diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py index 8827bcdad..7a233e0cc 100644 --- a/bbot/test/test_step_1/test_command.py +++ b/bbot/test/test_step_1/test_command.py @@ -102,25 +102,25 @@ async def test_command(bbot_scanner, bbot_config): path_parts = os.environ.get("PATH", "").split(":") assert "/tmp/.bbot_test/tools" in path_parts run_lines = (await scan1.helpers.run(["env"])).stdout.splitlines() - assert "BBOT_PLUMBUS=asdf" in run_lines + assert "BBOT_USER_AGENT=BBOT Test User-Agent" in run_lines for line in run_lines: if line.startswith("PATH="): path_parts = line.split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_lines_sudo = (await scan1.helpers.run(["env"], sudo=True)).stdout.splitlines() - assert "BBOT_PLUMBUS=asdf" in run_lines_sudo + assert "BBOT_USER_AGENT=BBOT Test User-Agent" in run_lines_sudo for line in run_lines_sudo: if line.startswith("PATH="): path_parts = line.split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_live_lines = [l async for l in scan1.helpers.run_live(["env"])] - assert "BBOT_PLUMBUS=asdf" in run_live_lines + assert "BBOT_USER_AGENT=BBOT Test User-Agent" in run_live_lines for line in run_live_lines: if line.startswith("PATH="): path_parts = line.strip().split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_live_lines_sudo = [l async for l in scan1.helpers.run_live(["env"], sudo=True)] - assert "BBOT_PLUMBUS=asdf" in run_live_lines_sudo + assert "BBOT_USER_AGENT=BBOT Test User-Agent" in run_live_lines_sudo for line in run_live_lines_sudo: if line.startswith("PATH="): path_parts = line.strip().split("=", 1)[-1].split(":") diff --git a/bbot/test/test_step_1/test_config.py b/bbot/test/test_step_1/test_config.py index 2d9980a2c..a526ccaf4 100644 --- a/bbot/test/test_step_1/test_config.py +++ b/bbot/test/test_step_1/test_config.py @@ -2,9 +2,20 @@ @pytest.mark.asyncio -async def test_config(bbot_config, bbot_scanner): - scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor", "speculate"], config=bbot_config) +async def test_config(bbot_scanner): + config = OmegaConf.create( + { + "plumbus": "asdf", + "modules": { + "ipneighbor": {"test_option": "ipneighbor"}, + "python": {"test_option": "asdf"}, + "speculate": {"test_option": "speculate"}, + }, + } + ) + scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor", "speculate"], config=config) await scan1.load_modules() + assert scan1.config.user_agent == "BBOT Test User-Agent" assert scan1.config.plumbus == "asdf" assert scan1.modules["ipneighbor"].config.test_option == "ipneighbor" assert scan1.modules["python"].config.test_option == "asdf" diff --git a/bbot/test/test_step_1/test_depsinstaller.py b/bbot/test/test_step_1/test_depsinstaller.py index 39a56bf41..e3f80d5cf 100644 --- a/bbot/test/test_step_1/test_depsinstaller.py +++ b/bbot/test/test_step_1/test_depsinstaller.py @@ -1,11 +1,9 @@ from ..bbot_fixtures import * -def test_depsinstaller(monkeypatch, bbot_config, bbot_scanner): +def test_depsinstaller(monkeypatch, bbot_scanner): scan = bbot_scanner( "127.0.0.1", - modules=["dnsresolve"], - config=bbot_config, ) # test shell diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index ecee203ec..39873ae0b 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -31,7 +31,7 @@ def bbot_other_httpservers(): @pytest.mark.asyncio -async def test_manager_scope_accuracy(bbot_config, bbot_scanner, bbot_httpserver, bbot_other_httpservers, bbot_httpserver_ssl, mock_dns): +async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_httpservers, bbot_httpserver_ssl, mock_dns): """ This test ensures that BBOT correctly handles different scope distance settings. It performs these tests for normal modules, output modules, and their graph variants, @@ -91,8 +91,7 @@ async def handle_batch(self, *events): self.events.append(event) async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs): - merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(_config)) - scan = bbot_scanner(*args, config=merged_config, **kwargs) + scan = bbot_scanner(*args, config=_config, **kwargs) dummy_module = DummyModule(scan) dummy_module_nodupes = DummyModuleNoDupes(scan) dummy_graph_output_module = DummyGraphOutputModule(scan) @@ -311,6 +310,7 @@ def custom_setup(scan): "scope_dns_search_distance": 2, "scope_report_distance": 1, "speculate": True, + "excavate": True, "modules": {"speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, @@ -329,6 +329,8 @@ def custom_setup(scan): assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) + return + assert len(all_events) == 14 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) @@ -782,17 +784,15 @@ def custom_setup(scan): @pytest.mark.asyncio -async def test_manager_blacklist(bbot_config, bbot_scanner, bbot_httpserver, caplog, mock_dns): +async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog, mock_dns): bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="") # dns search distance = 1, report distance = 0 - config = {"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0} - merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(config)) scan = bbot_scanner( "http://127.0.0.1:8888", - modules=["httpx", "excavate"], - config=merged_config, + modules=["httpx"], + config={"excavate": True, "dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0}, whitelist=["127.0.0.0/29", "test.notreal"], blacklist=["127.0.0.64/29"], ) @@ -810,8 +810,8 @@ async def test_manager_blacklist(bbot_config, bbot_scanner, bbot_httpserver, cap @pytest.mark.asyncio -async def test_manager_scope_tagging(bbot_config, bbot_scanner): - scan = bbot_scanner("test.notreal", config=bbot_config) +async def test_manager_scope_tagging(bbot_scanner): + scan = bbot_scanner("test.notreal") e1 = scan.make_event("www.test.notreal", source=scan.root_event, tags=["affiliate"]) assert e1.scope_distance == 1 assert "distance-1" in e1.tags diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index e61c22d77..bfebe786a 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -167,20 +167,24 @@ def test_preset_scope(): def test_preset_logging(): # test verbosity levels (conflicting verbose/debug/silent) preset = Preset(verbose=True) - assert preset.verbose == True - assert preset.debug == False - assert preset.silent == False - assert preset.core.logger.log_level == logging.VERBOSE - preset.debug = True - assert preset.verbose == False - assert preset.debug == True - assert preset.silent == False - assert preset.core.logger.log_level == logging.DEBUG - preset.silent = True - assert preset.verbose == False - assert preset.debug == False - assert preset.silent == True - assert preset.core.logger.log_level == logging.CRITICAL + original_log_level = preset.core.logger.log_level + try: + assert preset.verbose == True + assert preset.debug == False + assert preset.silent == False + assert preset.core.logger.log_level == logging.VERBOSE + preset.debug = True + assert preset.verbose == False + assert preset.debug == True + assert preset.silent == False + assert preset.core.logger.log_level == logging.DEBUG + preset.silent = True + assert preset.verbose == False + assert preset.debug == False + assert preset.silent == True + assert preset.core.logger.log_level == logging.CRITICAL + finally: + preset.core.logger.log_level = original_log_level def test_preset_module_resolution(): @@ -551,6 +555,17 @@ def test_preset_conditions(): Scanner(preset=preset) +def test_preset_internal_module_disablement(): + preset = Preset(config={"speculate": True, "excavate": True, "aggregate": True}).bake() + assert "speculate" in preset.internal_modules + assert "excavate" in preset.internal_modules + assert "aggregate" in preset.internal_modules + preset = Preset(config={"speculate": False, "excavate": True, "aggregate": True}).bake() + assert "speculate" not in preset.internal_modules + assert "excavate" in preset.internal_modules + assert "aggregate" in preset.internal_modules + + # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation # what if you specify -c modules.custommodule.option diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 00ad2d972..11e7a35ec 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -2,17 +2,17 @@ @pytest.mark.asyncio -async def test_python_api(bbot_config): +async def test_python_api(): from bbot.scanner import Scanner # make sure events are properly yielded - scan1 = Scanner("127.0.0.1", config=bbot_config) + scan1 = Scanner("127.0.0.1") events1 = [] async for event in scan1.async_start(): events1.append(event) assert any("127.0.0.1" == e for e in events1) # make sure output files work - scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") + scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") await scan2.async_start_without_generator() scan_home = scan2.helpers.scans_dir / "python_api_test" out_file = scan_home / "output.ndjson" @@ -24,7 +24,7 @@ async def test_python_api(bbot_config): assert debug_log.is_file() assert "python_api_test" in open(debug_log).read() - scan3 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="scan_logging_test") + scan3 = Scanner("127.0.0.1", output_modules=["json"], scan_name="scan_logging_test") await scan3.async_start_without_generator() assert "scan_logging_test" not in open(scan_log).read() @@ -46,17 +46,17 @@ async def test_python_api(bbot_config): assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools") -def test_python_api_sync(bbot_config): +def test_python_api_sync(): from bbot.scanner import Scanner # make sure events are properly yielded - scan1 = Scanner("127.0.0.1", config=bbot_config) + scan1 = Scanner("127.0.0.1") events1 = [] for event in scan1.start(): events1.append(event) assert any("127.0.0.1" == e for e in events1) # make sure output files work - scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") + scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") scan2.start_without_generator() out_file = scan2.helpers.scans_dir / "python_api_test" / "output.ndjson" assert list(scan2.helpers.read_file(out_file)) diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 985dc5c79..6e160b323 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -77,7 +77,7 @@ def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, mon *module_test_base.targets, modules=modules, output_modules=output_modules, - name=module_test_base._scan_name, + scan_name=module_test_base._scan_name, config=self.config, whitelist=module_test_base.whitelist, blacklist=module_test_base.blacklist, diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py index c115e0abe..9226d323d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_nmap.py +++ b/bbot/test/test_step_2/module_tests/test_module_nmap.py @@ -40,7 +40,9 @@ async def setup_after_prep(self, module_test): first_scan_config = module_test.scan.config.copy() first_scan_config["modules"]["asset_inventory"]["use_previous"] = False - first_scan = Scanner("127.0.0.1", name=self.scan_name, modules=["asset_inventory"], config=first_scan_config) + first_scan = Scanner( + "127.0.0.1", scan_name=self.scan_name, modules=["asset_inventory"], config=first_scan_config + ) await first_scan.async_start_without_generator() asset_inventory_output_file = first_scan.home / "asset-inventory.csv" diff --git a/docs/dev/index.md b/docs/dev/index.md index 093a3aefb..f2a333cb6 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -16,18 +16,10 @@ import discord from discord.ext import commands from bbot.scanner import Scanner -from bbot.modules import module_loader +from bbot.modules import MODULE_LOADER from bbot.modules.output.discord import Discord -# make list of BBOT modules to enable for the scan -bbot_modules = ["excavate", "speculate", "aggregate"] -for module_name, preloaded in module_loader.preloaded().items(): - flags = preloaded["flags"] - if "subdomain-enum" in flags and "passive" in flags and "slow" not in flags: - bbot_modules.append(module_name) - - class BBOTDiscordBot(commands.Cog): """ A simple Discord bot capable of running a BBOT scan. @@ -63,7 +55,7 @@ class BBOTDiscordBot(commands.Cog): await ctx.send(f"Starting scan against {target}.") # creates scan instance - self.current_scan = Scanner(target, modules=bbot_modules) + self.current_scan = Scanner(target, flags="subdomain-enum") discord_module = Discord(self.current_scan) seen = set() From 9de5048d25d6d85f3decbfdaa7829dae0cc6d54c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 21:43:12 -0400 Subject: [PATCH 060/171] tests stuff --- bbot/modules/output/splunk.py | 2 +- bbot/test/test_step_1/test_cloud_helpers.py | 12 ++++++------ bbot/test/test_step_1/test_config.py | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index 242f1759e..00d70876b 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -1,4 +1,4 @@ -from bbot.core.errors import RequestError +from httpx import RequestError from bbot.modules.output.base import BaseOutputModule diff --git a/bbot/test/test_step_1/test_cloud_helpers.py b/bbot/test/test_step_1/test_cloud_helpers.py index b42da11a7..adfc290ca 100644 --- a/bbot/test/test_step_1/test_cloud_helpers.py +++ b/bbot/test/test_step_1/test_cloud_helpers.py @@ -2,8 +2,8 @@ @pytest.mark.asyncio -async def test_cloud_helpers(bbot_scanner, bbot_config): - scan1 = bbot_scanner("127.0.0.1", config=bbot_config) +async def test_cloud_helpers(bbot_scanner): + scan1 = bbot_scanner("127.0.0.1") provider_names = ("amazon", "google", "azure", "digitalocean", "oracle", "akamai", "cloudflare", "github") for provider_name in provider_names: @@ -51,12 +51,12 @@ async def test_cloud_helpers(bbot_scanner, bbot_config): @pytest.mark.asyncio -async def test_cloud_helpers_excavate(bbot_scanner, bbot_config, bbot_httpserver): +async def test_cloud_helpers_excavate(bbot_scanner, bbot_httpserver): url = bbot_httpserver.url_for("/test_cloud_helpers_excavate") bbot_httpserver.expect_request(uri="/test_cloud_helpers_excavate").respond_with_data( "" ) - scan1 = bbot_scanner(url, modules=["httpx", "excavate"], config=bbot_config) + scan1 = bbot_scanner(url, modules=["httpx"], config={"excavate": True}) events = [e async for e in scan1.async_start()] assert 1 == len( [ @@ -71,8 +71,8 @@ async def test_cloud_helpers_excavate(bbot_scanner, bbot_config, bbot_httpserver @pytest.mark.asyncio -async def test_cloud_helpers_speculate(bbot_scanner, bbot_config): - scan1 = bbot_scanner("asdf.s3.amazonaws.com", modules=["speculate"], config=bbot_config) +async def test_cloud_helpers_speculate(bbot_scanner): + scan1 = bbot_scanner("asdf.s3.amazonaws.com", config={"speculate": True}) events = [e async for e in scan1.async_start()] assert 1 == len( [ diff --git a/bbot/test/test_step_1/test_config.py b/bbot/test/test_step_1/test_config.py index a526ccaf4..f62b11912 100644 --- a/bbot/test/test_step_1/test_config.py +++ b/bbot/test/test_step_1/test_config.py @@ -6,6 +6,7 @@ async def test_config(bbot_scanner): config = OmegaConf.create( { "plumbus": "asdf", + "speculate": True, "modules": { "ipneighbor": {"test_option": "ipneighbor"}, "python": {"test_option": "asdf"}, @@ -13,7 +14,7 @@ async def test_config(bbot_scanner): }, } ) - scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor", "speculate"], config=config) + scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor"], config=config) await scan1.load_modules() assert scan1.config.user_agent == "BBOT Test User-Agent" assert scan1.config.plumbus == "asdf" From 72e55b04ef7ce7099ae5c5ef790042112d419a29 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 22:12:50 -0400 Subject: [PATCH 061/171] and more tests --- bbot/test/bbot_fixtures.py | 10 ++--- .../test_step_1/test_manager_deduplication.py | 37 +++++++++++++++---- bbot/test/test_step_1/test_modules_basic.py | 16 +++----- bbot/test/test_step_2/module_tests/base.py | 6 +-- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index bd9dd68f1..a2f44fe5f 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -36,11 +36,11 @@ for h in root_logger.handlers: h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) -default_preset = Preset() +DEFAULT_PRESET = Preset() -available_modules = list(default_preset.module_loader.configs(type="scan")) -available_output_modules = list(default_preset.module_loader.configs(type="output")) -available_internal_modules = list(default_preset.module_loader.configs(type="internal")) +available_modules = list(DEFAULT_PRESET.module_loader.configs(type="scan")) +available_output_modules = list(DEFAULT_PRESET.module_loader.configs(type="output")) +available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal")) class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): @@ -230,7 +230,7 @@ def bbot_config(): @pytest.fixture(autouse=True) def install_all_python_deps(): deps_pip = set() - for module in default_preset.module_loader.preloaded().values(): + for module in DEFAULT_PRESET.module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index e8ec69edc..435e57991 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -3,7 +3,7 @@ @pytest.mark.asyncio -async def test_manager_deduplication(bbot_config, bbot_scanner, mock_dns): +async def test_manager_deduplication(bbot_scanner, mock_dns): class DefaultModule(BaseModule): _name = "default_module" @@ -47,8 +47,7 @@ class PerDomainOnly(DefaultModule): async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs): - merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(_config)) - scan = bbot_scanner(*args, config=merged_config, **kwargs) + scan = bbot_scanner(*args, config=_config, **kwargs) default_module = DefaultModule(scan) everything_module = EverythingModule(scan) no_suppress_dupes = NoSuppressDupes(scan) @@ -91,16 +90,40 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) + # SCAN / severe_jacqueline (SCAN:d233093d39044c961754c97f749fa758543d7474) + # DNS_NAME / test.notreal + # OPEN_TCP_PORT / default_module.test.notreal:88 + # OPEN_TCP_PORT / test.notreal:88 + # DNS_NAME / default_module.test.notreal + # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 + # OPEN_TCP_PORT / accept_dupes.test.notreal:88 + # DNS_NAME / per_hostport_only.test.notreal + # DNS_NAME / no_suppress_dupes.test.notreal + # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 + # DNS_NAME / no_suppress_dupes.test.notreal + # DNS_NAME / no_suppress_dupes.test.notreal + # DNS_NAME / accept_dupes.test.notreal + # OPEN_TCP_PORT / per_domain_only.test.notreal:88 + # DNS_NAME / no_suppress_dupes.test.notreal + # DNS_NAME / no_suppress_dupes.test.notreal + # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 + # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 + # DNS_NAME / per_domain_only.test.notreal + # OPEN_TCP_PORT / per_hostport_only.test.notreal:88 + # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 + + for e in events: + log.critical(f"{e.type} / {e.data} / {e.module} / {e.source.data} / {e.source.module}") assert len(events) == 21 - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"]) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 95a70afc1..30939d37e 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -9,7 +9,7 @@ @pytest.mark.asyncio -async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, httpx_mock): +async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock): fallback_nameservers = scan.helpers.temp_dir / "nameservers.txt" with open(fallback_nameservers, "w") as f: f.write("8.8.8.8\n") @@ -82,7 +82,6 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h scan2 = bbot_scanner( modules=list(set(available_modules + available_internal_modules)), output_modules=list(available_output_modules), - config=bbot_config, force_start=True, ) scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers @@ -104,7 +103,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h assert not any(not_async) # module preloading - all_preloaded = module_loader.preloaded() + all_preloaded = DEFAULT_PRESET.module_loader.preloaded() assert "massdns" in all_preloaded assert "DNS_NAME" in all_preloaded["massdns"]["watched_events"] assert "DNS_NAME" in all_preloaded["massdns"]["produced_events"] @@ -177,7 +176,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h @pytest.mark.asyncio -async def test_modules_basic_perhostonly(helpers, events, bbot_config, bbot_scanner, httpx_mock, monkeypatch): +async def test_modules_basic_perhostonly(helpers, events, bbot_scanner, httpx_mock, monkeypatch): from bbot.modules.base import BaseModule class mod_normal(BaseModule): @@ -201,7 +200,6 @@ class mod_domain_only(BaseModule): scan = bbot_scanner( "evilcorp.com", - config=bbot_config, force_start=True, ) @@ -265,11 +263,10 @@ class mod_domain_only(BaseModule): @pytest.mark.asyncio -async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, bbot_scanner, httpx_mock, monkeypatch): +async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_scanner, httpx_mock, monkeypatch): per_domain_scan = bbot_scanner( "evilcorp.com", modules=list(set(available_modules + available_internal_modules)), - config=bbot_config, force_start=True, ) @@ -306,7 +303,7 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, b @pytest.mark.asyncio -async def test_modules_basic_stats(helpers, events, bbot_config, bbot_scanner, httpx_mock, monkeypatch, mock_dns): +async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, monkeypatch, mock_dns): from bbot.modules.base import BaseModule class dummy(BaseModule): @@ -323,8 +320,7 @@ async def handle_event(self, event): scan = bbot_scanner( "evilcorp.com", - modules=["speculate"], - config=bbot_config, + config={"speculate": True}, force_start=True, ) mock_dns( diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 6e160b323..fb51bbf23 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -5,10 +5,9 @@ from omegaconf import OmegaConf from types import SimpleNamespace +from ...bbot_fixtures import * from bbot.scanner import Scanner -from bbot.core import CORE from bbot.core.helpers.misc import rand_string -from ...bbot_fixtures import test_config, MockResolver, bbot_test_dir log = logging.getLogger("bbot.test.modules") @@ -57,8 +56,7 @@ def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, mon self.httpserver_ssl = httpserver_ssl self.monkeypatch = monkeypatch self.request_fixture = request - # PRESET TODO: revisit this - self.preloaded = CORE.module_loader.preloaded() + self.preloaded = DEFAULT_PRESET.module_loader.preloaded() # handle output, internal module types output_modules = None From d1f7fa685b2e267286a31c300db93c3f57475d99 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 22:22:04 -0400 Subject: [PATCH 062/171] more tests --- bbot/core/flags.py | 1 + bbot/test/test_step_1/test_modules_basic.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bbot/core/flags.py b/bbot/core/flags.py index d8dbf8566..f65dbad28 100644 --- a/bbot/core/flags.py +++ b/bbot/core/flags.py @@ -4,6 +4,7 @@ "aggressive": "Generates a large amount of network traffic", "baddns": "Runs all modules from the DNS auditing tool BadDNS", "cloud-enum": "Enumerates cloud resources", + "code-enum": "Find public code repositories and search them for secrets etc.", "deadly": "Highly aggressive", "email-enum": "Enumerates email addresses", "iis-shortnames": "Scans for IIS Shortname vulnerability", diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 30939d37e..8f696c1ac 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -321,6 +321,7 @@ async def handle_event(self, event): scan = bbot_scanner( "evilcorp.com", config={"speculate": True}, + output_modules=["python"], force_start=True, ) mock_dns( From f15f2befd94d339e49c2ed71d64c35215fd312e6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 22:49:21 -0400 Subject: [PATCH 063/171] more tests --- bbot/core/modules.py | 12 ++++++------ bbot/test/test_step_1/test_presets.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index db34a39c6..4f2824229 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -130,13 +130,10 @@ def preload(self, module_dirs=None): # try to load from cache module_cache_key = (str(module_file), tuple(module_file.stat())) - preloaded_module = self.preload_cache.get(module_name, {}) - flags = preloaded_module.get("flags", []) - self._all_flags.update(set(flags)) - cache_key = preloaded_module.get("cache_key", ()) - if module_cache_key == cache_key: + preloaded = self.preload_cache.get(module_name, {}) + cache_key = preloaded.get("cache_key", ()) + if preloaded and module_cache_key == cache_key: log.debug(f"Preloading {module_name} from cache") - preloaded = self.preload_cache[module_name] else: log.debug(f"Preloading {module_name} from disk") if module_dir.name == "modules": @@ -168,6 +165,9 @@ def preload(self, module_dirs=None): log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") sys.exit(1) + flags = preloaded.get("flags", []) + self._all_flags.update(set(flags)) + self.__preloaded[module_name] = preloaded config = OmegaConf.create(preloaded.get("config", {})) self._configs[module_name] = config diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index bfebe786a..d65d169ab 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -94,6 +94,24 @@ def test_preset_yaml(): assert yaml1 == yaml2 +# yaml_string_1 = """ +# flags: +# - subdomain-enum + +# modules: +# - wappalyzer + +# output_modules: +# - csv + +# config: +# speculate: False +# """ +# preset3 = Preset.from_yaml_string(yaml_string_1) +# yaml_string_2 = preset3.to_yaml(sort_keys=True) +# assert yaml_string_2 == yaml_string + + def test_preset_scope(): blank_preset = Preset() From 98858162fcc00ce11a6d69ad1af0c61490e38436 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 23:11:17 -0400 Subject: [PATCH 064/171] fix cloud bucket tests --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 1623dc700..0ec91ce67 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -393,7 +393,7 @@ async def handle_event(self, event): module = None provider = cloud_kwargs.pop("_provider", "") if provider: - module = self.scan._make_dummy_module(provider) + module = self.scan._make_dummy_module(f"{provider}_cloud") await self.emit_event(module=module, **cloud_kwargs) await self.search( From 4730fcea9936de3933cf25b6e3b5c06c85910f0e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 26 Mar 2024 23:52:47 -0400 Subject: [PATCH 065/171] fix excavate tests --- bbot/core/event/base.py | 2 +- bbot/scanner/manager.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 793015967..0d65cca04 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -883,7 +883,7 @@ class URL_UNVERIFIED(BaseEvent): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # increment the web spider distance - if self.type == "URL_UNVERIFIED" and getattr(self.module, "name", "") != "TARGET": + if self.type == "URL_UNVERIFIED": self.web_spider_distance += 1 self.num_redirects = getattr(self.source, "num_redirects", 0) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index c5c74caba..c3f15a20f 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -68,6 +68,7 @@ async def init_events(self): for event in sorted_events: event._dummy = False event.scope_distance = 0 + event.web_spider_distance = 0 event.scan = self.scan event.source = self.scan.root_event event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") From 983ccd67a7eb0b5fc02b348df50eab3bd0643518 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 27 Mar 2024 11:43:58 -0400 Subject: [PATCH 066/171] fix ffuf shortnames tests --- bbot/modules/ffuf_shortnames.py | 100 +++++++++++++++----------------- bbot/scanner/manager.py | 6 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index 562ae681f..ca02da886 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -92,8 +92,7 @@ async def setup(self): self.extensions = parse_list_string(self.config.get("extensions", "")) self.debug(f"Using custom extensions: [{','.join(self.extensions)}]") except ValueError as e: - self.warning(f"Error parsing extensions: {e}") - return False + return False, f"Error parsing extensions: {e}" self.ignore_redirects = self.config.get("ignore_redirects") @@ -123,78 +122,73 @@ def find_delimiter(self, hint): return None async def filter_event(self, event): + if event.source.type != "URL": + return False, "its source event is not of type URL" return True async def handle_event(self, event): - if event.source.type == "URL": - filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower() + filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower() - host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/" - if host not in self.per_host_collection.keys(): - self.per_host_collection[host] = [(filename_hint, event.source.data)] + host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/" + if host not in self.per_host_collection.keys(): + self.per_host_collection[host] = [(filename_hint, event.source.data)] - else: - self.per_host_collection[host].append((filename_hint, event.source.data)) + else: + self.per_host_collection[host].append((filename_hint, event.source.data)) - self.shortname_to_event[filename_hint] = event + self.shortname_to_event[filename_hint] = event - root_stub = "/".join(event.parsed.path.split("/")[:-1]) - root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/" + root_stub = "/".join(event.parsed.path.split("/")[:-1]) + root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/" - if "shortname-file" in event.tags: - used_extensions = self.build_extension_list(event) - - if len(filename_hint) == 6: - tempfile, tempfile_len = self.generate_templist(prefix=filename_hint) - self.verbose( - f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]" - ) - - else: - tempfile = self.helpers.tempfile([filename_hint], pipe=False) - tempfile_len = 1 - - if tempfile_len > 0: - if "shortname-file" in event.tags: - for ext in used_extensions: - async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): - await self.emit_event( - r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] - ) + if "shortname-file" in event.tags: + used_extensions = self.build_extension_list(event) - elif "shortname-directory" in event.tags: - async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): - r_url = f"{r['url'].rstrip('/')}/" - await self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + if len(filename_hint) == 6: + tempfile, tempfile_len = self.generate_templist(prefix=filename_hint) + self.verbose( + f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]" + ) - if self.config.get("find_delimiters"): - if "shortname-directory" in event.tags: + else: + tempfile = self.helpers.tempfile([filename_hint], pipe=False) + tempfile_len = 1 + + if tempfile_len > 0: + if "shortname-file" in event.tags: + for ext in used_extensions: + async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): + await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + elif "shortname-directory" in event.tags: + async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): + r_url = f"{r['url'].rstrip('/')}/" + await self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + if self.config.get("find_delimiters"): + if "shortname-directory" in event.tags: + delimiter_r = self.find_delimiter(filename_hint) + if delimiter_r: + delimiter, prefix, partial_hint = delimiter_r + self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]") + tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) + async for r in self.execute_ffuf(tempfile, root_url, prefix=f"{prefix}{delimiter}", exts=["/"]): + await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + elif "shortname-file" in event.tags: + for ext in used_extensions: delimiter_r = self.find_delimiter(filename_hint) if delimiter_r: delimiter, prefix, partial_hint = delimiter_r self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]") tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) async for r in self.execute_ffuf( - tempfile, root_url, prefix=f"{prefix}{delimiter}", exts=["/"] + tempfile, root_url, prefix=f"{prefix}{delimiter}", suffix=f".{ext}" ): await self.emit_event( r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] ) - elif "shortname-file" in event.tags: - for ext in used_extensions: - delimiter_r = self.find_delimiter(filename_hint) - if delimiter_r: - delimiter, prefix, partial_hint = delimiter_r - self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]") - tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) - async for r in self.execute_ffuf( - tempfile, root_url, prefix=f"{prefix}{delimiter}", suffix=f".{ext}" - ): - await self.emit_event( - r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] - ) - async def finish(self): if self.config.get("find_common_prefixes"): per_host_collection = dict(self.per_host_collection) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index c3f15a20f..86a2f224a 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -70,8 +70,10 @@ async def init_events(self): event.scope_distance = 0 event.web_spider_distance = 0 event.scan = self.scan - event.source = self.scan.root_event - event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") + if event.source is None: + event.source = self.scan.root_event + if event.module is None: + event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") self.scan.verbose(f"Target: {event}") self.queue_event(event) await asyncio.sleep(0.1) From 3b6192a4ab691a0b3d6dedbfeb03110811a67055 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 27 Mar 2024 14:06:20 -0400 Subject: [PATCH 067/171] fix nmap, github_org tests --- bbot/scanner/scanner.py | 1 + bbot/test/bbot_fixtures.py | 9 ----- .../module_tests/test_module_github_org.py | 33 ++++++++++++++++--- .../module_tests/test_module_nmap.py | 9 +++-- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c1dd1bd14..33065f377 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -131,6 +131,7 @@ def __init__( self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" preset = preset_kwargs.pop("preset", None) + preset_kwargs["_log"] = True if preset is None: preset = Preset(*args, **preset_kwargs) self.preset = preset.bake() diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index a2f44fe5f..267d017f3 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -213,15 +213,6 @@ class bbot_events: return bbot_events -@pytest.fixture -def agent(monkeypatch, bbot_config): - from bbot import agent - - test_agent = agent.Agent(bbot_config) - test_agent.setup() - return test_agent - - @pytest.fixture def bbot_config(): return test_config diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py index 4a01544f6..fd0e3ea5b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_org.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py @@ -280,7 +280,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ e @@ -298,10 +298,22 @@ def check(self, module_test, events): if e.type == "SOCIAL" and e.data["platform"] == "github" and e.data["profile_name"] == "blacklanternsecurity" + and str(e.module) == "github_org" and "github-org" in e.tags and e.scope_distance == 1 ] ), "Failed to find blacklanternsecurity github" + assert 1 == len( + [ + e + for e in events + if e.type == "SOCIAL" + and e.data["platform"] == "github" + and e.data["profile_name"] == "blacklanternsecurity" + and str(e.module) == "social" + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity github (social module)" assert 1 == len( [ e @@ -309,6 +321,7 @@ def check(self, module_test, events): if e.type == "SOCIAL" and e.data["platform"] == "github" and e.data["profile_name"] == "TheTechromancer" + and str(e.module) == "github_org" and "github-org-member" in e.tags and e.scope_distance == 2 ] @@ -329,7 +342,7 @@ class TestGithub_Org_No_Members(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_members": False}}} def check(self, module_test, events): - assert len(events) == 5 + assert len(events) == 6 assert 1 == len( [ e @@ -337,10 +350,22 @@ def check(self, module_test, events): if e.type == "SOCIAL" and e.data["platform"] == "github" and e.data["profile_name"] == "blacklanternsecurity" + and str(e.module) == "github_org" and "github-org" in e.tags and e.scope_distance == 1 ] ), "Failed to find blacklanternsecurity github" + assert 1 == len( + [ + e + for e in events + if e.type == "SOCIAL" + and e.data["platform"] == "github" + and e.data["profile_name"] == "blacklanternsecurity" + and str(e.module) == "social" + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity github (social module)" assert 0 == len( [ e @@ -356,7 +381,7 @@ class TestGithub_Org_MemberRepos(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_member_repos": True}}} def check(self, module_test, events): - assert len(events) == 7 + assert len(events) == 8 assert 1 == len( [ e @@ -366,4 +391,4 @@ def check(self, module_test, events): and e.data["url"] == "https://github.com/TheTechromancer/websitedemo" and e.scope_distance == 2 ] - ), "Found to find TheTechromancer github repo" + ), "Failed to find TheTechromancer github repo" diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py index 9226d323d..32b5495e5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_nmap.py +++ b/bbot/test/test_step_2/module_tests/test_module_nmap.py @@ -38,10 +38,13 @@ class TestNmapAssetInventory(ModuleTestBase): async def setup_after_prep(self, module_test): from bbot.scanner import Scanner - first_scan_config = module_test.scan.config.copy() - first_scan_config["modules"]["asset_inventory"]["use_previous"] = False first_scan = Scanner( - "127.0.0.1", scan_name=self.scan_name, modules=["asset_inventory"], config=first_scan_config + "127.0.0.1", + scan_name=self.scan_name, + output_modules=["asset_inventory"], + config={ + "modules": {"nmap": {"ports": "8888,8889"}, "asset_inventory": {"use_previous": False}}, + }, ) await first_scan.async_start_without_generator() From 91c621c8bd472a7f3e20bd506409f675b50f38cd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 27 Mar 2024 17:55:27 -0400 Subject: [PATCH 068/171] more preset tests --- bbot/cli.py | 54 +++++----- bbot/core/config/logger.py | 7 +- bbot/scanner/preset/args.py | 8 +- bbot/scanner/preset/preset.py | 77 +++++++------- bbot/test/bbot_fixtures.py | 28 ++--- bbot/test/conftest.py | 8 ++ bbot/test/test_step_1/test_cli.py | 86 +++++++++++++++- bbot/test/test_step_1/test_command.py | 4 +- bbot/test/test_step_1/test_dns.py | 12 +-- bbot/test/test_step_1/test_events.py | 2 +- bbot/test/test_step_1/test_files.py | 4 +- bbot/test/test_step_1/test_helpers.py | 6 +- bbot/test/test_step_1/test_presets.py | 141 +++++++++++++++++++++++--- bbot/test/test_step_1/test_scan.py | 14 +-- bbot/test/test_step_1/test_target.py | 12 +-- bbot/test/test_step_1/test_web.py | 39 ++++--- 16 files changed, 344 insertions(+), 158 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 7d6c4ce26..34408ddbb 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -43,14 +43,6 @@ async def _main(): # ensure arguments (-c config options etc.) are valid options = preset.args.parsed - modules_to_list = None - if options.modules or options.output_modules: - modules_to_list = set() - if options.modules: - modules_to_list.update(set(preset.scan_modules)) - if options.output_modules: - modules_to_list.update(set(preset.output_modules)) - # print help if no arguments if len(sys.argv) == 1: preset.args.parser.print_help() @@ -72,23 +64,37 @@ async def _main(): print(row) return - # --list-modules - if options.list_modules: - print("") - print("### MODULES ###") - print("") - for row in preset.module_loader.modules_table(modules_to_list).splitlines(): - print(row) - return + if options.list_modules or options.list_module_options: - # --list-module-options - if options.list_module_options: - print("") - print("### MODULE OPTIONS ###") - print("") - for row in preset.module_loader.modules_options_table(modules=modules_to_list).splitlines(): - print(row) - return + modules_to_list = set() + if options.modules or options.flags: + modules_to_list.update(set(preset.scan_modules)) + if options.output_modules: + modules_to_list.update(set(preset.output_modules)) + + if not (options.modules or options.output_modules or options.flags): + for module, preloaded in preset.module_loader.preloaded().items(): + module_type = preloaded.get("type", "scan") + preset.add_module(module, module_type=module_type) + modules_to_list.update(set(preset.modules)) + + # --list-modules + if options.list_modules: + print("") + print("### MODULES ###") + print("") + for row in preset.module_loader.modules_table(modules_to_list).splitlines(): + print(row) + return + + # --list-module-options + if options.list_module_options: + print("") + print("### MODULE OPTIONS ###") + print("") + for row in preset.module_loader.modules_options_table(modules_to_list).splitlines(): + print(row) + return # --list-flags if options.list_flags: diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 03af61de8..31ca4184e 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -59,12 +59,7 @@ def __init__(self, core): for logger in self.loggers: self.include_logger(logger) - if core.config.get("verbose", False): - self.log_level = logging.VERBOSE - elif core.config.get("debug", False): - self.log_level = logging.DEBUG - else: - self.log_level = logging.INFO + self.log_level = logging.INFO def addLoggingLevel(self, levelName, levelNum, methodName=None): """ diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 0e24dd0d0..45d532854 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -125,15 +125,15 @@ def preset_from_args(self): self.validate() # load modules & flags (excluded then required then all others) - args_preset.exclude_modules = self.parsed.exclude_modules - args_preset.exclude_flags = self.parsed.exclude_flags - args_preset.require_flags = self.parsed.require_flags + args_preset.add_excluded_modules(self.parsed.exclude_modules) + args_preset.add_excluded_flags(self.parsed.exclude_flags) + args_preset.add_required_flags(self.parsed.require_flags) for scan_module in self.parsed.modules: args_preset.add_module(scan_module, module_type="scan") for output_module in self.parsed.output_modules: args_preset.add_module(output_module, module_type="output") - args_preset.flags = self.parsed.flags + args_preset.add_flags(self.parsed.flags) # dependencies if self.parsed.retry_deps: diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index e23bd3524..f595c6065 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -32,7 +32,6 @@ def __init__( modules=None, output_modules=None, exclude_modules=None, - internal_modules=None, flags=None, require_flags=None, exclude_flags=None, @@ -50,7 +49,7 @@ def __init__( conditions=None, force=False, _exclude=None, - _log=False, + _log=True, ): self._log = _log self.scan = None @@ -62,6 +61,9 @@ def __init__( self._modules = set() + self.explicit_scan_modules = set() if modules is None else set(modules) + self.explicit_output_modules = set() if output_modules is None else set(output_modules) + self._exclude_modules = set() self._require_flags = set() self._exclude_flags = set() @@ -132,8 +134,6 @@ def __init__( modules = [] if output_modules is None: output_modules = ["python", "csv", "human", "json"] - if internal_modules is None: - internal_modules = ["aggregate", "excavate", "speculate"] if isinstance(modules, str): modules = [modules] if isinstance(output_modules, str): @@ -142,8 +142,15 @@ def __init__( self.add_required_flags(require_flags if require_flags is not None else []) self.add_excluded_flags(exclude_flags if exclude_flags is not None else []) self.add_scan_modules(modules if modules is not None else []) - self.add_output_modules(output_modules if output_modules is not None else []) - self.add_internal_modules(internal_modules if internal_modules is not None else []) + self.add_output_modules(output_modules) + + # add internal modules + for internal_module, preloaded in self.module_loader.preloaded(type="internal").items(): + is_enabled = self.config.get(internal_module, True) + is_excluded = internal_module in self.exclude_modules + if is_enabled and not is_excluded: + self.add_module(internal_module, module_type="internal") + self.add_flags(flags if flags is not None else []) @property @@ -155,6 +162,7 @@ def preset_dir(self): return self.bbot_home / "presets" def merge(self, other): + self.log_debug(f'Merging preset "{other.name}" into "{self.name}"') # config self.core.merge_custom(other.core.custom_config) self.module_loader.core = self.core @@ -165,6 +173,8 @@ def merge(self, other): self.add_required_flags(other.require_flags) self.add_excluded_flags(other.exclude_flags) # then it's okay to start enabling modules + self.explicit_scan_modules.update(other.explicit_scan_modules) + self.explicit_output_modules.update(other.explicit_output_modules) self.add_flags(other.flags) for module_name in other.modules: module_type = self.preloaded_module(module_name).get("type", "scan") @@ -253,6 +263,7 @@ def modules(self, modules): if isinstance(modules, str): modules = [modules] modules = set(modules) + modules.update(self.internal_modules) for module_name in modules: self.add_module(module_name) @@ -278,11 +289,6 @@ def output_modules(self, modules): def internal_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] - @internal_modules.setter - def internal_modules(self, modules): - self.log_debug(f"Setting internal modules to {modules}") - self._modules_setter(modules, module_type="internal") - def _modules_setter(self, modules, module_type="scan"): if isinstance(modules, str): modules = [modules] @@ -306,10 +312,12 @@ def add_internal_modules(self, modules): self.add_module(module, module_type="internal") def add_module(self, module_name, module_type="scan"): - # log.info(f'Adding "{module_name}": {module_type}') if module_name in self.exclude_modules: self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') return + if module_name in self.modules: + self.log_debug(f'Already added module "{module_name}"') + return try: preloaded = self.module_loader.preloaded()[module_name] except KeyError: @@ -329,19 +337,18 @@ def add_module(self, module_name, module_type="scan"): if f in self.exclude_flags: self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') return - if self.require_flags and not any(f in self.require_flags for f in module_flags): + if self.require_flags and not all(f in module_flags for f in self.require_flags): self.log_verbose( f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' ) return - if module_name not in self.modules: - self.log_debug(f'Adding module "{module_name}"') - self.modules.add(module_name) - for module_dep in preloaded.get("deps", {}).get("modules", []): - if module_dep not in self.modules: - self.log_verbose(f'Adding module "{module_dep}" because {module_name} depends on it') - self.add_module(module_dep) + self.log_debug(f'Adding module "{module_name}"') + self._modules.add(module_name) + for module_dep in preloaded.get("deps", {}).get("modules", []): + if module_dep != module_name and module_dep not in self.modules: + self.log_verbose(f'Adding module "{module_dep}" because {module_name} depends on it') + self.add_module(module_dep) @property def exclude_modules(self): @@ -362,17 +369,24 @@ def flags(self): @flags.setter def flags(self, flags): self.log_debug(f"Setting flags to {flags}") - self._flags = set(flags) - for flag in flags: - self.add_flag(flag) + flags = set(flags) + self._flags = flags + self.add_flags(flags) def add_flags(self, flags): - for flag in flags: - self.add_flag(flag) + if flags: + self.log_debug(f"Adding flags: {flags}") + for flag in flags: + self.add_flag(flag) def add_flag(self, flag): if not flag in self.module_loader._all_flags: raise EnableFlagError(f'Flag "{flag}" was not found') + if flag in self.exclude_flags: + self.log_debug(f'Skipping flag "{flag}" because it\'s excluded') + return + self.log_debug(f'Adding flag "{flag}"') + self._flags.add(flag) for module, preloaded in self.module_loader.preloaded().items(): module_flags = preloaded.get("flags", []) if flag in module_flags: @@ -456,7 +470,6 @@ def verbose(self, value): if value: self._debug = False self._silent = False - self.core.merge_custom({"verbose": True}) self.core.logger.log_level = "VERBOSE" else: with suppress(omegaconf.errors.ConfigKeyError): @@ -473,7 +486,6 @@ def debug(self, value): if value: self._verbose = False self._silent = False - self.core.merge_custom({"debug": True}) self.core.logger.log_level = "DEBUG" else: with suppress(omegaconf.errors.ConfigKeyError): @@ -490,7 +502,6 @@ def silent(self, value): if value: self._verbose = False self._debug = False - self.core.merge_custom({"silent": True}) self.core.logger.log_level = "CRITICAL" else: with suppress(omegaconf.errors.ConfigKeyError): @@ -666,12 +677,10 @@ def to_dict(self, include_target=False, full_config=False): preset_dict["exclude_modules"] = sorted(self.exclude_modules) if self.flags: preset_dict["flags"] = sorted(self.flags) - scan_modules = self.scan_modules - output_modules = self.output_modules - if scan_modules: - preset_dict["modules"] = sorted(scan_modules) - if output_modules: - preset_dict["output_modules"] = sorted(output_modules) + if self.explicit_scan_modules: + preset_dict["modules"] = sorted(self.explicit_scan_modules) + if self.explicit_output_modules: + preset_dict["output_modules"] = sorted(self.explicit_output_modules) # log verbosity if self.verbose: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 267d017f3..4365bb9d3 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -8,7 +8,6 @@ import tldextract import pytest_httpserver from pathlib import Path -from omegaconf import OmegaConf from werkzeug.wrappers import Request @@ -25,17 +24,6 @@ # bbot config -test_config = OmegaConf.load(Path(__file__).parent / "test.conf") -CORE.merge_custom(test_config) - -if test_config.get("debug", False): - logging.getLogger("bbot").setLevel(logging.DEBUG) -else: - # silence stdout + trace - root_logger = logging.getLogger() - for h in root_logger.handlers: - h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) - DEFAULT_PRESET = Preset() available_modules = list(DEFAULT_PRESET.module_loader.configs(type="scan")) @@ -43,6 +31,13 @@ available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal")) +@pytest.fixture +def clean_default_config(monkeypatch): + clean_config = CORE.files_config.get_default_config() + monkeypatch.setattr("bbot.core.core.DEFAULT_CONFIG", clean_config) + yield + + class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): def match_data(self, request: Request) -> bool: if self.data is None: @@ -69,10 +64,10 @@ def bbot_scanner(): @pytest.fixture -def scan(monkeypatch, bbot_config): +def scan(monkeypatch): from bbot.scanner import Scanner - bbot_scan = Scanner("127.0.0.1", modules=["ipneighbor"], config=bbot_config) + bbot_scan = Scanner("127.0.0.1", modules=["ipneighbor"]) fallback_nameservers_file = bbot_scan.helpers.bbot_home / "fallback_nameservers.txt" with open(fallback_nameservers_file, "w") as f: @@ -213,11 +208,6 @@ class bbot_events: return bbot_events -@pytest.fixture -def bbot_config(): - return test_config - - @pytest.fixture(autouse=True) def install_all_python_deps(): deps_pip = set() diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 3b5775b2e..3dd403106 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -17,6 +17,14 @@ if test_config.get("debug", False): os.environ["BBOT_DEBUG"] = "True" +if test_config.get("debug", False): + logging.getLogger("bbot").setLevel(logging.DEBUG) +else: + # silence stdout + trace + root_logger = logging.getLogger() + for h in root_logger.handlers: + h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) + CORE.merge_default(test_config) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index bfe504268..11b7a10fa 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -82,6 +82,29 @@ async def test_cli_args(monkeypatch, capsys): assert "| bool" in captured.out assert "| emit URLs in addition to DNS_NAMEs" in captured.out assert "| False" in captured.out + assert "| modules.massdns.wordlist" in captured.out + assert "| modules.robots.include_allow" in captured.out + + # list module options by flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "--list-module-options"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert "| modules.wayback.urls" in captured.out + assert "| bool" in captured.out + assert "| emit URLs in addition to DNS_NAMEs" in captured.out + assert "| False" in captured.out + assert "| modules.massdns.wordlist" in captured.out + assert not "| modules.robots.include_allow" in captured.out + + # list module options by module + monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdns", "-lmo"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert not "| modules.wayback.urls" in captured.out + assert "| modules.massdns.wordlist" in captured.out + assert not "| modules.robots.include_allow" in captured.out # list flags monkeypatch.setattr("sys.argv", ["bbot", "--list-flags"]) @@ -90,6 +113,26 @@ async def test_cli_args(monkeypatch, capsys): captured = capsys.readouterr() assert "| safe" in captured.out assert "| Non-intrusive, safe to run" in captured.out + assert "| active" in captured.out + assert "| passive" in captured.out + + # list only a single flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "--list-flags"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert not "| safe" in captured.out + assert "| active" in captured.out + assert not "| passive" in captured.out + + # list multiple flags + monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "safe", "--list-flags"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert "| safe" in captured.out + assert "| active" in captured.out + assert not "| passive" in captured.out # no args monkeypatch.setattr("sys.argv", ["bbot"]) @@ -98,10 +141,47 @@ async def test_cli_args(monkeypatch, capsys): captured = capsys.readouterr() assert "Target:\n -t TARGET [TARGET ...]" in captured.out - # enable module by flag - monkeypatch.setattr("sys.argv", ["bbot", "-f", "report"]) + # list modules + monkeypatch.setattr("sys.argv", ["bbot", "-l"]) result = await cli._main() - assert result == True + assert result == None + captured = capsys.readouterr() + assert "| massdns" in captured.out + assert "| httpx" in captured.out + assert "| robots" in captured.out + + # list modules by flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-l"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert "| massdns" in captured.out + assert "| httpx" in captured.out + assert not "| robots" in captured.out + + # list modules by flag + required flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-rf", "passive", "-l"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert "| massdns" in captured.out + assert not "| httpx" in captured.out + + # list modules by flag + excluded flag + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-ef", "active", "-l"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert "| massdns" in captured.out + assert not "| httpx" in captured.out + + # list modules by flag + excluded module + monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "massdns", "-l"]) + result = await cli._main() + assert result == None + captured = capsys.readouterr() + assert not "| massdns" in captured.out + assert "| httpx" in captured.out # unconsoleable output module monkeypatch.setattr("sys.argv", ["bbot", "-om", "web_report"]) diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py index 7a233e0cc..a3772cefe 100644 --- a/bbot/test/test_step_1/test_command.py +++ b/bbot/test/test_step_1/test_command.py @@ -3,8 +3,8 @@ @pytest.mark.asyncio -async def test_command(bbot_scanner, bbot_config): - scan1 = bbot_scanner(config=bbot_config) +async def test_command(bbot_scanner): + scan1 = bbot_scanner() # run assert "plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"])).stdout diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 91465507e..6fd51800f 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -2,8 +2,8 @@ @pytest.mark.asyncio -async def test_dns(bbot_scanner, bbot_config, mock_dns): - scan = bbot_scanner("1.1.1.1", config=bbot_config) +async def test_dns(bbot_scanner, mock_dns): + scan = bbot_scanner("1.1.1.1") helpers = scan.helpers # lowest level functions @@ -82,9 +82,7 @@ async def test_dns(bbot_scanner, bbot_config, mock_dns): assert "1.1.1.1" in [str(x) for x in children2["A"]] assert set(children1.keys()) == set(children2.keys()) - dns_config = OmegaConf.create({"dns_resolution": True}) - dns_config = OmegaConf.merge(bbot_config, dns_config) - scan2 = bbot_scanner("evilcorp.com", config=dns_config) + scan2 = bbot_scanner("evilcorp.com", config={"dns_resolution": True}) mock_dns( scan2, { @@ -99,8 +97,8 @@ async def test_dns(bbot_scanner, bbot_config, mock_dns): @pytest.mark.asyncio -async def test_wildcards(bbot_scanner, bbot_config): - scan = bbot_scanner("1.1.1.1", config=bbot_config) +async def test_wildcards(bbot_scanner): + scan = bbot_scanner("1.1.1.1") helpers = scan.helpers # wildcards diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 024ee84a3..26f963474 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -6,7 +6,7 @@ @pytest.mark.asyncio -async def test_events(events, scan, helpers, bbot_config): +async def test_events(events, scan, helpers): assert events.ipv4.type == "IP_ADDRESS" assert events.ipv6.type == "IP_ADDRESS" assert events.netv4.type == "IP_RANGE" diff --git a/bbot/test/test_step_1/test_files.py b/bbot/test/test_step_1/test_files.py index be52b1cd2..ed9bc0a33 100644 --- a/bbot/test/test_step_1/test_files.py +++ b/bbot/test/test_step_1/test_files.py @@ -4,8 +4,8 @@ @pytest.mark.asyncio -async def test_files(bbot_scanner, bbot_config): - scan1 = bbot_scanner(config=bbot_config) +async def test_files(bbot_scanner): + scan1 = bbot_scanner() # tempfile tempfile = scan1.helpers.tempfile(("line1", "line2"), pipe=False) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 4780eb522..1b793711a 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -6,7 +6,7 @@ @pytest.mark.asyncio -async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): +async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): ### URL ### bad_urls = ( "http://e.co/index.html", @@ -452,7 +452,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https ) -def test_word_cloud(helpers, bbot_config, bbot_scanner): +def test_word_cloud(helpers, bbot_scanner): number_mutations = helpers.word_cloud.get_number_mutations("base2_p013", n=5, padding=2) assert "base0_p013" in number_mutations assert "base7_p013" in number_mutations @@ -468,7 +468,7 @@ def test_word_cloud(helpers, bbot_config, bbot_scanner): assert ("dev", "_base") in permutations # saving and loading - scan1 = bbot_scanner("127.0.0.1", config=bbot_config) + scan1 = bbot_scanner("127.0.0.1") word_cloud = scan1.helpers.word_cloud word_cloud.add_word("lantern") word_cloud.add_word("black") diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index d65d169ab..445f991b8 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -57,7 +57,9 @@ def test_core(): assert "test456" in core_copy.config["test123"] -def test_preset_yaml(): +def test_preset_yaml(clean_default_config): + + import yaml preset1 = Preset( "evilcorp.com", @@ -93,23 +95,41 @@ def test_preset_yaml(): yaml2 = preset2.to_yaml(sort_keys=True) assert yaml1 == yaml2 + yaml_string_1 = """ +flags: + - subdomain-enum + +exclude_flags: + - aggressive + - slow -# yaml_string_1 = """ -# flags: -# - subdomain-enum +require_flags: + - passive + - safe -# modules: -# - wappalyzer +exclude_modules: + - certspotter + - rapiddns -# output_modules: -# - csv +modules: + - robots + - wappalyzer -# config: -# speculate: False -# """ -# preset3 = Preset.from_yaml_string(yaml_string_1) -# yaml_string_2 = preset3.to_yaml(sort_keys=True) -# assert yaml_string_2 == yaml_string +output_modules: + - csv + - json + +config: + speculate: False + excavate: True +""" + yaml_string_1 = yaml.dump(yaml.safe_load(yaml_string_1), sort_keys=True) + # preset from yaml + preset3 = Preset.from_yaml_string(yaml_string_1) + # yaml to preset + yaml_string_2 = preset3.to_yaml(sort_keys=True) + # make sure they're the same + assert yaml_string_2 == yaml_string_1 def test_preset_scope(): @@ -205,7 +225,7 @@ def test_preset_logging(): preset.core.logger.log_level = original_log_level -def test_preset_module_resolution(): +def test_preset_module_resolution(clean_default_config): preset = Preset() sslcert_preloaded = preset.preloaded_module("sslcert") wayback_preloaded = preset.preloaded_module("wayback") @@ -440,6 +460,9 @@ class TestModule4(BaseModule): assert "testmodule3" in preset2.module_loader.preloaded() assert "testmodule4" in preset2.module_loader.preloaded() + # reset module_loader + preset2.module_loader.__init__() + def test_preset_include(): @@ -584,6 +607,94 @@ def test_preset_internal_module_disablement(): assert "aggregate" in preset.internal_modules +def test_preset_require_exclude(): + + def get_module_flags(p): + for m in p.scan_modules: + preloaded = p.preloaded_module(m) + yield m, preloaded.get("flags", []) + + # enable by flag, no exclusions/requirements + preset = Preset(flags=["subdomain-enum"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + massdns_flags = preset.preloaded_module("massdns").get("flags", []) + assert "subdomain-enum" in massdns_flags + assert "passive" in massdns_flags + assert not "active" in massdns_flags + assert "aggressive" in massdns_flags + assert not "safe" in massdns_flags + assert "massdns" in [x[0] for x in module_flags] + assert "certspotter" in [x[0] for x in module_flags] + assert "c99" in [x[0] for x in module_flags] + assert any("passive" in flags for module, flags in module_flags) + assert any("active" in flags for module, flags in module_flags) + assert any("safe" in flags for module, flags in module_flags) + assert any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, one required flag + preset = Preset(flags=["subdomain-enum"], require_flags=["passive"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert "massdns" in [x[0] for x in module_flags] + assert all("passive" in flags for module, flags in module_flags) + assert not any("active" in flags for module, flags in module_flags) + assert any("safe" in flags for module, flags in module_flags) + assert any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, one excluded flag + preset = Preset(flags=["subdomain-enum"], exclude_flags=["active"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert "massdns" in [x[0] for x in module_flags] + assert all("passive" in flags for module, flags in module_flags) + assert not any("active" in flags for module, flags in module_flags) + assert any("safe" in flags for module, flags in module_flags) + assert any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, one excluded module + preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert not "massdns" in [x[0] for x in module_flags] + assert any("passive" in flags for module, flags in module_flags) + assert any("active" in flags for module, flags in module_flags) + assert any("safe" in flags for module, flags in module_flags) + assert any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, multiple required flags + preset = Preset(flags=["subdomain-enum"], require_flags=["safe", "passive"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert not "massdns" in [x[0] for x in module_flags] + assert all("passive" in flags and "safe" in flags for module, flags in module_flags) + assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags) + assert not any("active" in flags for module, flags in module_flags) + assert not any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, multiple excluded flags + preset = Preset(flags=["subdomain-enum"], exclude_flags=["aggressive", "active"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert not "massdns" in [x[0] for x in module_flags] + assert all("passive" in flags and "safe" in flags for module, flags in module_flags) + assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags) + assert not any("active" in flags for module, flags in module_flags) + assert not any("aggressive" in flags for module, flags in module_flags) + + # enable by flag, multiple excluded modules + preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns", "c99"]) + assert len(preset.modules) > 25 + module_flags = list(get_module_flags(preset)) + assert not "massdns" in [x[0] for x in module_flags] + assert "certspotter" in [x[0] for x in module_flags] + assert not "c99" in [x[0] for x in module_flags] + assert any("passive" in flags for module, flags in module_flags) + assert any("active" in flags for module, flags in module_flags) + assert any("safe" in flags for module, flags in module_flags) + assert any("aggressive" in flags for module, flags in module_flags) + + # test custom module load directory # make sure it works with cli arg module/flag/config syntax validation # what if you specify -c modules.custommodule.option diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 464f2038b..5775deaf1 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -4,7 +4,6 @@ @pytest.mark.asyncio async def test_scan( events, - bbot_config, helpers, monkeypatch, bbot_scanner, @@ -15,7 +14,6 @@ async def test_scan( "evilcorp.com", blacklist=["1.1.1.1/28", "www.evilcorp.com"], modules=["ipneighbor"], - config=bbot_config, ) await scan0.load_modules() assert scan0.whitelisted("1.1.1.1") @@ -38,7 +36,7 @@ async def test_scan( assert "1.1.1.0/28" in j["blacklist"] assert "ipneighbor" in j["modules"] - scan1 = bbot_scanner("1.1.1.1", whitelist=["1.0.0.1"], config=bbot_config) + scan1 = bbot_scanner("1.1.1.1", whitelist=["1.0.0.1"]) assert not scan1.blacklisted("1.1.1.1") assert not scan1.blacklisted("1.0.0.1") assert not scan1.whitelisted("1.1.1.1") @@ -46,7 +44,7 @@ async def test_scan( assert scan1.in_scope("1.0.0.1") assert not scan1.in_scope("1.1.1.1") - scan2 = bbot_scanner("1.1.1.1", config=bbot_config) + scan2 = bbot_scanner("1.1.1.1") assert not scan2.blacklisted("1.1.1.1") assert not scan2.blacklisted("1.0.0.1") assert scan2.whitelisted("1.1.1.1") @@ -60,9 +58,7 @@ async def test_scan( } # make sure DNS resolution works - dns_config = OmegaConf.create({"dns_resolution": True}) - dns_config = OmegaConf.merge(bbot_config, dns_config) - scan4 = bbot_scanner("1.1.1.1", config=dns_config) + scan4 = bbot_scanner("1.1.1.1", config={"dns_resolution": True}) mock_dns(scan4, dns_table) events = [] async for event in scan4.async_start(): @@ -71,9 +67,7 @@ async def test_scan( assert "one.one.one.one" in event_data # make sure it doesn't work when you turn it off - no_dns_config = OmegaConf.create({"dns_resolution": False}) - no_dns_config = OmegaConf.merge(bbot_config, no_dns_config) - scan5 = bbot_scanner("1.1.1.1", config=no_dns_config) + scan5 = bbot_scanner("1.1.1.1", config={"dns_resolution": True}) mock_dns(scan5, dns_table) events = [] async for event in scan5.async_start(): diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 90db526c7..dced8af02 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -1,12 +1,12 @@ from ..bbot_fixtures import * # noqa: F401 -def test_target(bbot_config, bbot_scanner): - scan1 = bbot_scanner("api.publicapis.org", "8.8.8.8/30", "2001:4860:4860::8888/126", config=bbot_config) - scan2 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125", config=bbot_config) - scan3 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125", config=bbot_config) - scan4 = bbot_scanner("8.8.8.8/29", config=bbot_config) - scan5 = bbot_scanner(config=bbot_config) +def test_target(bbot_scanner): + scan1 = bbot_scanner("api.publicapis.org", "8.8.8.8/30", "2001:4860:4860::8888/126") + scan2 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125") + scan3 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125") + scan4 = bbot_scanner("8.8.8.8/29") + scan5 = bbot_scanner() assert not scan5.target assert len(scan1.target) == 9 assert len(scan4.target) == 8 diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index 675197265..c6654d4d4 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -1,17 +1,16 @@ import re -from omegaconf import OmegaConf from ..bbot_fixtures import * @pytest.mark.asyncio -async def test_web_helpers(bbot_scanner, bbot_config, bbot_httpserver): - scan1 = bbot_scanner("8.8.8.8", config=bbot_config) - scan2 = bbot_scanner("127.0.0.1", config=bbot_config) +async def test_web_helpers(bbot_scanner, bbot_httpserver): + scan1 = bbot_scanner("8.8.8.8") + scan2 = bbot_scanner("127.0.0.1") - user_agent = bbot_config.get("user_agent", "") + user_agent = test_config.get("user_agent", "") headers = {"User-Agent": user_agent} - custom_headers = bbot_config.get("http_headers", {}) + custom_headers = test_config.get("http_headers", {}) headers.update(custom_headers) assert headers["test"] == "header" @@ -125,7 +124,7 @@ async def test_web_helpers(bbot_scanner, bbot_config, bbot_httpserver): @pytest.mark.asyncio -async def test_web_interactsh(bbot_scanner, bbot_config, bbot_httpserver): +async def test_web_interactsh(bbot_scanner, bbot_httpserver): from bbot.core.helpers.interactsh import server_list sync_called = False @@ -134,7 +133,7 @@ async def test_web_interactsh(bbot_scanner, bbot_config, bbot_httpserver): sync_correct_url = False async_correct_url = False - scan1 = bbot_scanner("8.8.8.8", config=bbot_config) + scan1 = bbot_scanner("8.8.8.8") scan1.status = "RUNNING" interactsh_client = scan1.helpers.interactsh(poll_interval=3) @@ -186,8 +185,8 @@ def sync_callback(data): @pytest.mark.asyncio -async def test_web_curl(bbot_scanner, bbot_config, bbot_httpserver): - scan = bbot_scanner("127.0.0.1", config=bbot_config) +async def test_web_curl(bbot_scanner, bbot_httpserver): + scan = bbot_scanner("127.0.0.1") helpers = scan.helpers url = bbot_httpserver.url_for("/curl") bbot_httpserver.expect_request(uri="/curl").respond_with_data("curl_yep") @@ -231,7 +230,7 @@ async def test_web_http_compare(httpx_mock, helpers): @pytest.mark.asyncio -async def test_http_proxy(bbot_scanner, bbot_config, bbot_httpserver, proxy_server): +async def test_http_proxy(bbot_scanner, bbot_httpserver, proxy_server): endpoint = "/test_http_proxy" url = bbot_httpserver.url_for(endpoint) # test user agent + custom headers @@ -239,9 +238,7 @@ async def test_http_proxy(bbot_scanner, bbot_config, bbot_httpserver, proxy_serv proxy_address = f"http://127.0.0.1:{proxy_server.server_address[1]}" - test_config = OmegaConf.merge(bbot_config, OmegaConf.create({"http_proxy": proxy_address})) - - scan = bbot_scanner("127.0.0.1", config=test_config) + scan = bbot_scanner("127.0.0.1", config={"http_proxy": proxy_address}) assert len(proxy_server.RequestHandlerClass.urls) == 0 @@ -256,17 +253,15 @@ async def test_http_proxy(bbot_scanner, bbot_config, bbot_httpserver, proxy_serv @pytest.mark.asyncio -async def test_http_ssl(bbot_scanner, bbot_config, bbot_httpserver_ssl): +async def test_http_ssl(bbot_scanner, bbot_httpserver_ssl): endpoint = "/test_http_ssl" url = bbot_httpserver_ssl.url_for(endpoint) # test user agent + custom headers bbot_httpserver_ssl.expect_request(uri=endpoint).respond_with_data("test_http_ssl_yep") - verify_config = OmegaConf.merge(bbot_config, OmegaConf.create({"ssl_verify": True, "http_debug": True})) - scan1 = bbot_scanner("127.0.0.1", config=verify_config) + scan1 = bbot_scanner("127.0.0.1", config={"ssl_verify": True, "http_debug": True}) - not_verify_config = OmegaConf.merge(bbot_config, OmegaConf.create({"ssl_verify": False, "http_debug": True})) - scan2 = bbot_scanner("127.0.0.1", config=not_verify_config) + scan2 = bbot_scanner("127.0.0.1", config={"ssl_verify": False, "http_debug": True}) r1 = await scan1.helpers.request(url) assert r1 is None, "Request to self-signed SSL server went through even with ssl_verify=True" @@ -276,12 +271,12 @@ async def test_http_ssl(bbot_scanner, bbot_config, bbot_httpserver_ssl): @pytest.mark.asyncio -async def test_web_cookies(bbot_scanner, bbot_config, httpx_mock): +async def test_web_cookies(bbot_scanner, httpx_mock): import httpx # make sure cookies work when enabled httpx_mock.add_response(url="http://www.evilcorp.com/cookies", headers=[("set-cookie", "wat=asdf; path=/")]) - scan = bbot_scanner(config=bbot_config) + scan = bbot_scanner() client = scan.helpers.AsyncClient(persist_cookies=True) r = await client.get(url="http://www.evilcorp.com/cookies") assert r.cookies["wat"] == "asdf" @@ -294,7 +289,7 @@ async def test_web_cookies(bbot_scanner, bbot_config, httpx_mock): # make sure they don't when they're not httpx_mock.add_response(url="http://www2.evilcorp.com/cookies", headers=[("set-cookie", "wats=fdsa; path=/")]) - scan = bbot_scanner(config=bbot_config) + scan = bbot_scanner() client2 = scan.helpers.AsyncClient(persist_cookies=False) r = await client2.get(url="http://www2.evilcorp.com/cookies") # make sure we can access the cookies From f8392b7a22639bc8fb2da09dac3960a36269872b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 00:00:48 -0400 Subject: [PATCH 069/171] fix tests --- bbot/test/bbot_fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 4365bb9d3..3e32c134d 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -8,6 +8,7 @@ import tldextract import pytest_httpserver from pathlib import Path +from omegaconf import OmegaConf # noqa from werkzeug.wrappers import Request From 79039ded2aaf1f5bb4a3d1b7db49a0ee99668830 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 00:43:51 -0400 Subject: [PATCH 070/171] fix scan tests --- bbot/test/test_step_1/test_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 5775deaf1..3f1c01c04 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -67,7 +67,7 @@ async def test_scan( assert "one.one.one.one" in event_data # make sure it doesn't work when you turn it off - scan5 = bbot_scanner("1.1.1.1", config={"dns_resolution": True}) + scan5 = bbot_scanner("1.1.1.1", config={"dns_resolution": False}) mock_dns(scan5, dns_table) events = [] async for event in scan5.async_start(): From 73b0602451d7ceeb84283bc471d1e2f13fcc4f21 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 01:59:07 -0400 Subject: [PATCH 071/171] steady work on tests --- bbot/cli.py | 36 ++-- bbot/scanner/preset/args.py | 7 +- bbot/scanner/preset/preset.py | 7 +- bbot/test/bbot_fixtures.py | 9 +- bbot/test/test_step_1/test_cli.py | 250 ++++++++++++++++---------- bbot/test/test_step_1/test_presets.py | 21 ++- 6 files changed, 207 insertions(+), 123 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 34408ddbb..0f77615d5 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -45,23 +45,23 @@ async def _main(): # print help if no arguments if len(sys.argv) == 1: - preset.args.parser.print_help() + log.stdout(preset.args.parser.format_help()) sys.exit(1) return # --version if options.version: - print(__version__) + log.stdout(__version__) sys.exit(0) return # --list-presets if options.list_presets: - print("") - print("### PRESETS ###") - print("") + log.stdout("") + log.stdout("### PRESETS ###") + log.stdout("") for row in preset.presets_table().splitlines(): - print(row) + log.stdout(row) return if options.list_modules or options.list_module_options: @@ -80,30 +80,30 @@ async def _main(): # --list-modules if options.list_modules: - print("") - print("### MODULES ###") - print("") + log.stdout("") + log.stdout("### MODULES ###") + log.stdout("") for row in preset.module_loader.modules_table(modules_to_list).splitlines(): - print(row) + log.stdout(row) return # --list-module-options if options.list_module_options: - print("") - print("### MODULE OPTIONS ###") - print("") + log.stdout("") + log.stdout("### MODULE OPTIONS ###") + log.stdout("") for row in preset.module_loader.modules_options_table(modules_to_list).splitlines(): - print(row) + log.stdout(row) return # --list-flags if options.list_flags: flags = preset.flags if preset.flags else None - print("") - print("### FLAGS ###") - print("") + log.stdout("") + log.stdout("### FLAGS ###") + log.stdout("") for row in preset.module_loader.flags_table(flags=flags).splitlines(): - print(row) + log.stdout(row) return deadly_modules = [m for m in preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", [])] diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 45d532854..73bc63773 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -77,6 +77,7 @@ def __init__(self, preset): # validate module choices self._module_choices = sorted(set(self.preset.module_loader.preloaded(type="scan"))) self._output_module_choices = sorted(set(self.preset.module_loader.preloaded(type="output"))) + self._all_module_choices = sorted(set(self.preset.module_loader.preloaded())) self._flag_choices = set() for m, c in self.preset.module_loader.preloaded().items(): self._flag_choices.update(set(c.get("flags", []))) @@ -335,10 +336,10 @@ def validate(self): # validate modules for m in self.parsed.modules: if m not in self._module_choices: - raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="module")) + raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="scan module")) for m in self.parsed.exclude_modules: - if m not in self._module_choices: - raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="module")) + if m not in self._all_module_choices: + raise BBOTArgumentError(get_closest_match(m, self._all_module_choices, msg="module")) for m in self.parsed.output_modules: if m not in self._output_module_choices: raise BBOTArgumentError(get_closest_match(m, self._output_module_choices, msg="output module")) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index f595c6065..0244c40b0 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -133,7 +133,7 @@ def __init__( if modules is None: modules = [] if output_modules is None: - output_modules = ["python", "csv", "human", "json"] + output_modules = [] if isinstance(modules, str): modules = [modules] if isinstance(output_modules, str): @@ -229,6 +229,11 @@ def bake(self): if baked_preset.config.get(internal_module, True) == False: baked_preset.exclude_module(internal_module) + # ensure we have output modules + if not self.output_modules: + for output_module in ("python", "csv", "human", "json"): + self.add_module(output_module, module_type="output") + # evaluate conditions if baked_preset.conditions: from .conditions import ConditionEvaluator diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 3e32c134d..085107b52 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -34,9 +34,12 @@ @pytest.fixture def clean_default_config(monkeypatch): - clean_config = CORE.files_config.get_default_config() - monkeypatch.setattr("bbot.core.core.DEFAULT_CONFIG", clean_config) - yield + clean_config = OmegaConf.merge( + CORE.files_config.get_default_config(), {"modules": DEFAULT_PRESET.module_loader.configs()} + ) + with monkeypatch.context() as m: + m.setattr("bbot.core.core.DEFAULT_CONFIG", clean_config) + yield class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 11b7a10fa..a33bd1c7a 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -2,7 +2,7 @@ @pytest.mark.asyncio -async def test_cli_args(monkeypatch, capsys): +async def test_cli_scan(monkeypatch): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) @@ -16,7 +16,8 @@ async def test_cli_args(monkeypatch, capsys): "argv", ["bbot", "-y", "-t", "127.0.0.1", "www.example.com", "-n", "test_cli_scan", "-c", "dns_resolution=False"], ) - await cli._main() + result = await cli._main() + assert result == True scan_home = scans_home / "test_cli_scan" assert (scan_home / "wordcloud.tsv").is_file(), "wordcloud.tsv not found" @@ -40,148 +41,189 @@ async def test_cli_args(monkeypatch, capsys): dns_success = True assert ip_success and dns_success, "IP_ADDRESS and/or DNS_NAME are not present in output.txt" + +@pytest.mark.asyncio +async def test_cli_args(monkeypatch, caplog, clean_default_config): + from bbot import cli + + caplog.set_level(logging.INFO) + + monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) + monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + # show version + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "--version"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert captured.out.count(".") > 1 + assert len(caplog.text.splitlines()) == 1 + assert caplog.text.count(".") > 1 - # show current preset - monkeypatch.setattr("sys.argv", ["bbot", "-c", "http_proxy=currentpresettest", "--current-preset"]) + # output modules override + caplog.clear() + assert not caplog.text + monkeypatch.setattr("sys.argv", ["bbot", "-om", "csv,json", "-y"]) result = await cli._main() - assert result == None - captured = capsys.readouterr() - assert " http_proxy: currentpresettest" in captured.out + assert result == True + assert "Loaded 2/2 output modules, (csv,json)" in caplog.text + caplog.clear() + monkeypatch.setattr("sys.argv", ["bbot", "-em", "csv,json", "-y"]) + result = await cli._main() + assert result == True + assert "Loaded 2/2 output modules, (human,python)" in caplog.text - # show current preset (full) - monkeypatch.setattr("sys.argv", ["bbot", "--current-preset-full"]) + # internal modules override + caplog.clear() + assert not caplog.text + monkeypatch.setattr("sys.argv", ["bbot", "-y"]) result = await cli._main() - assert result == None - captured = capsys.readouterr() - assert " api_key: ''" in captured.out + assert result == True + assert "Loaded 3/3 internal modules (aggregate,excavate,speculate)" in caplog.text + caplog.clear() + monkeypatch.setattr("sys.argv", ["bbot", "-em", "excavate", "speculate", "-y"]) + result = await cli._main() + assert result == True + assert "Loaded 1/1 internal modules (aggregate)" in caplog.text + caplog.clear() + monkeypatch.setattr("sys.argv", ["bbot", "-c", "speculate=false", "-y"]) + result = await cli._main() + assert result == True + assert "Loaded 2/2 internal modules (aggregate,excavate)" in caplog.text # list modules + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"]) result = await cli._main() assert result == None - captured = capsys.readouterr() # internal modules - assert "| excavate" in captured.out + assert "| excavate" in caplog.text # output modules - assert "| csv" in captured.out + assert "| csv" in caplog.text # scan modules - assert "| wayback" in captured.out + assert "| wayback" in caplog.text # list module options + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| modules.wayback.urls" in captured.out - assert "| bool" in captured.out - assert "| emit URLs in addition to DNS_NAMEs" in captured.out - assert "| False" in captured.out - assert "| modules.massdns.wordlist" in captured.out - assert "| modules.robots.include_allow" in captured.out + assert "| modules.wayback.urls" in caplog.text + assert "| bool" in caplog.text + assert "| emit URLs in addition to DNS_NAMEs" in caplog.text + assert "| False" in caplog.text + assert "| modules.massdns.wordlist" in caplog.text + assert "| modules.robots.include_allow" in caplog.text # list module options by flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "--list-module-options"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| modules.wayback.urls" in captured.out - assert "| bool" in captured.out - assert "| emit URLs in addition to DNS_NAMEs" in captured.out - assert "| False" in captured.out - assert "| modules.massdns.wordlist" in captured.out - assert not "| modules.robots.include_allow" in captured.out + assert "| modules.wayback.urls" in caplog.text + assert "| bool" in caplog.text + assert "| emit URLs in addition to DNS_NAMEs" in caplog.text + assert "| False" in caplog.text + assert "| modules.massdns.wordlist" in caplog.text + assert not "| modules.robots.include_allow" in caplog.text # list module options by module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdns", "-lmo"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert not "| modules.wayback.urls" in captured.out - assert "| modules.massdns.wordlist" in captured.out - assert not "| modules.robots.include_allow" in captured.out + assert not "| modules.wayback.urls" in caplog.text + assert "| modules.massdns.wordlist" in caplog.text + assert not "| modules.robots.include_allow" in caplog.text # list flags + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "--list-flags"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| safe" in captured.out - assert "| Non-intrusive, safe to run" in captured.out - assert "| active" in captured.out - assert "| passive" in captured.out + assert "| safe" in caplog.text + assert "| Non-intrusive, safe to run" in caplog.text + assert "| active" in caplog.text + assert "| passive" in caplog.text # list only a single flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "--list-flags"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert not "| safe" in captured.out - assert "| active" in captured.out - assert not "| passive" in captured.out + assert not "| safe" in caplog.text + assert "| active" in caplog.text + assert not "| passive" in caplog.text # list multiple flags + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "safe", "--list-flags"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| safe" in captured.out - assert "| active" in captured.out - assert not "| passive" in captured.out + assert "| safe" in caplog.text + assert "| active" in caplog.text + assert not "| passive" in caplog.text # no args + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "Target:\n -t TARGET [TARGET ...]" in captured.out + assert "Target:\n -t TARGET [TARGET ...]" in caplog.text # list modules + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-l"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| massdns" in captured.out - assert "| httpx" in captured.out - assert "| robots" in captured.out + assert "| massdns" in caplog.text + assert "| httpx" in caplog.text + assert "| robots" in caplog.text # list modules by flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-l"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| massdns" in captured.out - assert "| httpx" in captured.out - assert not "| robots" in captured.out + assert "| massdns" in caplog.text + assert "| httpx" in caplog.text + assert not "| robots" in caplog.text # list modules by flag + required flag + caplog.clear() monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-rf", "passive", "-l"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| massdns" in captured.out - assert not "| httpx" in captured.out + assert "| massdns" in caplog.text + assert not "| httpx" in caplog.text # list modules by flag + excluded flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-ef", "active", "-l"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert "| massdns" in captured.out - assert not "| httpx" in captured.out + assert "| massdns" in caplog.text + assert not "| httpx" in caplog.text # list modules by flag + excluded module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "massdns", "-l"]) result = await cli._main() assert result == None - captured = capsys.readouterr() - assert not "| massdns" in captured.out - assert "| httpx" in captured.out + assert not "| massdns" in caplog.text + assert "| httpx" in caplog.text # unconsoleable output module monkeypatch.setattr("sys.argv", ["bbot", "-om", "web_report"]) @@ -229,74 +271,82 @@ async def test_cli_args(monkeypatch, capsys): # assert success, "--install-all-deps failed for at least one module" -def test_cli_config_validation(monkeypatch, capsys): +def test_cli_config_validation(monkeypatch, caplog): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) # incorrect module option + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-c", "modules.ipnegibhor.num_bits=4"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find module option "modules.ipnegibhor.num_bits"' in captured.err - assert 'Did you mean "modules.ipneighbor.num_bits"?' in captured.err + assert 'Could not find module option "modules.ipnegibhor.num_bits"' in caplog.text + assert 'Did you mean "modules.ipneighbor.num_bits"?' in caplog.text # incorrect global option + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-c", "web_spier_distance=4"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find module option "web_spier_distance"' in captured.err - assert 'Did you mean "web_spider_distance"?' in captured.err + assert 'Could not find module option "web_spier_distance"' in caplog.text + assert 'Did you mean "web_spider_distance"?' in caplog.text -def test_cli_module_validation(monkeypatch, capsys): +def test_cli_module_validation(monkeypatch, caplog): from bbot import cli monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) # incorrect module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdnss"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find module "massdnss"' in captured.err - assert 'Did you mean "massdns"?' in captured.err + assert 'Could not find scan module "massdnss"' in caplog.text + assert 'Did you mean "massdns"?' in caplog.text # incorrect excluded module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-em", "massdnss"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find module "massdnss"' in captured.err - assert 'Did you mean "massdns"?' in captured.err + assert 'Could not find module "massdnss"' in caplog.text + assert 'Did you mean "massdns"?' in caplog.text # incorrect output module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-om", "neoo4j"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find output module "neoo4j"' in captured.err - assert 'Did you mean "neo4j"?' in captured.err + assert 'Could not find output module "neoo4j"' in caplog.text + assert 'Did you mean "neo4j"?' in caplog.text # incorrect flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomainenum"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find flag "subdomainenum"' in captured.err - assert 'Did you mean "subdomain-enum"?' in captured.err + assert 'Could not find flag "subdomainenum"' in caplog.text + assert 'Did you mean "subdomain-enum"?' in caplog.text # incorrect excluded flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-ef", "subdomainenum"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find flag "subdomainenum"' in captured.err - assert 'Did you mean "subdomain-enum"?' in captured.err + assert 'Could not find flag "subdomainenum"' in caplog.text + assert 'Did you mean "subdomain-enum"?' in caplog.text # incorrect required flag + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-rf", "subdomainenum"]) cli.main() - captured = capsys.readouterr() - assert 'Could not find flag "subdomainenum"' in captured.err - assert 'Did you mean "subdomain-enum"?' in captured.err + assert 'Could not find flag "subdomainenum"' in caplog.text + assert 'Did you mean "subdomain-enum"?' in caplog.text def test_cli_presets(monkeypatch, capsys): @@ -306,6 +356,18 @@ def test_cli_presets(monkeypatch, capsys): monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + # show current preset + monkeypatch.setattr("sys.argv", ["bbot", "-c", "http_proxy=currentpresettest", "--current-preset"]) + cli.main() + captured = capsys.readouterr() + assert " http_proxy: currentpresettest" in captured.out + + # show current preset (full) + monkeypatch.setattr("sys.argv", ["bbot", "-c" "modules.c99.api_key=asdf", "--current-preset-full"]) + cli.main() + captured = capsys.readouterr() + assert " api_key: asdf" in captured.out + preset_dir = bbot_test_dir / "test_cli_presets" preset_dir.mkdir(exist_ok=True) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 445f991b8..97a76430a 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -226,7 +226,7 @@ def test_preset_logging(): def test_preset_module_resolution(clean_default_config): - preset = Preset() + preset = Preset().bake() sslcert_preloaded = preset.preloaded_module("sslcert") wayback_preloaded = preset.preloaded_module("wayback") wappalyzer_preloaded = preset.preloaded_module("wappalyzer") @@ -596,15 +596,28 @@ def test_preset_conditions(): Scanner(preset=preset) -def test_preset_internal_module_disablement(): - preset = Preset(config={"speculate": True, "excavate": True, "aggregate": True}).bake() +def test_preset_module_disablement(clean_default_config): + # internal module disablement + preset = Preset().bake() assert "speculate" in preset.internal_modules assert "excavate" in preset.internal_modules assert "aggregate" in preset.internal_modules - preset = Preset(config={"speculate": False, "excavate": True, "aggregate": True}).bake() + preset = Preset(config={"speculate": False}).bake() assert "speculate" not in preset.internal_modules assert "excavate" in preset.internal_modules assert "aggregate" in preset.internal_modules + preset = Preset(exclude_modules=["speculate", "excavate"]).bake() + assert "speculate" not in preset.internal_modules + assert "excavate" not in preset.internal_modules + assert "aggregate" in preset.internal_modules + + # internal module disablement + preset = Preset().bake() + assert set(preset.output_modules) == {"python", "human", "csv", "json"} + preset = Preset(exclude_modules=["human", "csv"]).bake() + assert set(preset.output_modules) == {"python", "json"} + preset = Preset(output_modules=["json"]).bake() + assert set(preset.output_modules) == {"json"} def test_preset_require_exclude(): From 251656a1c5addb157c7b8786b3e805ec531f0226 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 02:17:09 -0400 Subject: [PATCH 072/171] fix tests --- bbot/presets/kitchen-sink.yml | 1 + bbot/presets/web-screenshots.yml | 14 ++++++++++++++ bbot/test/bbot_fixtures.py | 1 - bbot/test/test_step_1/test_web.py | 4 ++-- bbot/test/test_step_2/module_tests/base.py | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 bbot/presets/web-screenshots.yml diff --git a/bbot/presets/kitchen-sink.yml b/bbot/presets/kitchen-sink.yml index afe1ee92c..542483180 100644 --- a/bbot/presets/kitchen-sink.yml +++ b/bbot/presets/kitchen-sink.yml @@ -9,3 +9,4 @@ include: - web-basic - paramminer - dirbust-light + - web-screenshots diff --git a/bbot/presets/web-screenshots.yml b/bbot/presets/web-screenshots.yml new file mode 100644 index 000000000..4641e7be3 --- /dev/null +++ b/bbot/presets/web-screenshots.yml @@ -0,0 +1,14 @@ +description: Take screenshots of webpages + +flags: + - web-screenshots + +config: + modules: + gowitness: + resolution_x: 1440 + resolution_y: 900 + # folder to output web screenshots (default is inside ~/.bbot/scans/scan_name) + output_path: "" + # whether to take screenshots of social media pages + social: True diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 085107b52..5185e83d0 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -23,7 +23,6 @@ bbot_test_dir = Path("/tmp/.bbot_test") mkdir(bbot_test_dir) -# bbot config DEFAULT_PRESET = Preset() diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index c6654d4d4..14da286d0 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -8,9 +8,9 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver): scan1 = bbot_scanner("8.8.8.8") scan2 = bbot_scanner("127.0.0.1") - user_agent = test_config.get("user_agent", "") + user_agent = CORE.config.get("user_agent", "") headers = {"User-Agent": user_agent} - custom_headers = test_config.get("http_headers", {}) + custom_headers = CORE.config.get("http_headers", {}) headers.update(custom_headers) assert headers["test"] == "header" diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index fb51bbf23..a4ea06f81 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -49,7 +49,7 @@ class ModuleTestBase: class ModuleTest: def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, monkeypatch, request): self.name = module_test_base.name - self.config = OmegaConf.merge(test_config, OmegaConf.create(module_test_base.config_overrides)) + self.config = OmegaConf.merge(CORE.config, OmegaConf.create(module_test_base.config_overrides)) self.httpx_mock = httpx_mock self.httpserver = httpserver From 84ff604fec1d16dc18b6db1a5a72b7a3afd82852 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 10:36:17 -0400 Subject: [PATCH 073/171] more tests --- bbot/scanner/preset/path.py | 4 +++- bbot/test/test_step_1/test_cli.py | 22 +++++++++++++++++++++- bbot/test/test_step_1/test_presets.py | 21 ++++++++++----------- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index 0a6b5a46b..00de89569 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -40,7 +40,9 @@ def find(self, filename): log.verbose(f'Found preset matching "{filename}" at {file}') self.add_path(file.parent) return file.resolve() - raise BBOTError(f'Could not find preset at "{filename}" - file does not exist') + raise BBOTError( + f'Could not find preset at "{filename}" - file does not exist. Use -lp to list available presets' + ) def __str__(self): return ":".join([str(s) for s in self.paths]) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index a33bd1c7a..8343e783b 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -60,6 +60,19 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): assert len(caplog.text.splitlines()) == 1 assert caplog.text.count(".") > 1 + # output dir and scan name + output_dir = bbot_test_dir / "bbot_cli_args_output" + scan_name = "bbot_cli_args_scan_name" + scan_dir = output_dir / scan_name + assert not output_dir.exists() + monkeypatch.setattr("sys.argv", ["bbot", "-o", str(output_dir), "-n", scan_name, "-y"]) + result = await cli._main() + assert result == True + assert output_dir.is_dir() + assert scan_dir.is_dir() + assert "[SCAN]" in open(scan_dir / "output.txt").read() + assert "[INFO]" in open(scan_dir / "scan.log").read() + # output modules override caplog.clear() assert not caplog.text @@ -349,7 +362,7 @@ def test_cli_module_validation(monkeypatch, caplog): assert 'Did you mean "subdomain-enum"?' in caplog.text -def test_cli_presets(monkeypatch, capsys): +def test_cli_presets(monkeypatch, capsys, caplog): import yaml from bbot import cli @@ -431,3 +444,10 @@ def test_cli_presets(monkeypatch, capsys): captured = capsys.readouterr() stdout_preset = yaml.safe_load(captured.out) assert stdout_preset["config"]["http_proxy"] == "asdf" + + # invalid preset + caplog.clear() + assert not caplog.text + monkeypatch.setattr("sys.argv", ["bbot", "-p", "asdfasdfasdf", "-y"]) + cli.main() + assert "file does not exist. Use -lp to list available presets" in caplog.text diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 97a76430a..03c01b61d 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -3,6 +3,16 @@ from bbot.scanner import Scanner, Preset +# FUTURE TODO: +# Consider testing possible edge cases: +# make sure custom module load directory works with cli arg module/flag/config syntax validation +# what if you specify -c modules.custommodule.option? +# the validation needs to not happen until after your custom preset preset has been loaded +# what if you specify flags in one preset, but another preset (loaded later) has more custom modules that match that flag? +# how do we make sure those other modules get loaded too? +# what if you specify a flag that's only on custom modules? Will it be rejected as invalid? + + def test_preset_descriptions(): # ensure very preset has a description preset = Preset() @@ -706,14 +716,3 @@ def get_module_flags(p): assert any("active" in flags for module, flags in module_flags) assert any("safe" in flags for module, flags in module_flags) assert any("aggressive" in flags for module, flags in module_flags) - - -# test custom module load directory -# make sure it works with cli arg module/flag/config syntax validation -# what if you specify -c modules.custommodule.option -# the validation needs to not happen until after all presets have been loaded -# what if you specify flags in one preset -# but another preset (loaded later) has more custom modules that match that flag -# what if you specify a flag that's only on custom modules? Will it be rejected as invalid? - -# cli test: nonexistent / invalid preset From 43b3967a515925a5b7359b4e89b6f86dafa634e3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 10:41:44 -0400 Subject: [PATCH 074/171] more presets --- bbot/presets/{web_advanced => web}/dirbust-heavy.yml | 0 bbot/presets/{web_advanced => web}/dirbust-light.yml | 5 +---- bbot/presets/web/iis-shortnames.yml | 10 ++++++++++ bbot/presets/{web_advanced => web}/paramminer.yml | 0 4 files changed, 11 insertions(+), 4 deletions(-) rename bbot/presets/{web_advanced => web}/dirbust-heavy.yml (100%) rename bbot/presets/{web_advanced => web}/dirbust-light.yml (54%) create mode 100644 bbot/presets/web/iis-shortnames.yml rename bbot/presets/{web_advanced => web}/paramminer.yml (100%) diff --git a/bbot/presets/web_advanced/dirbust-heavy.yml b/bbot/presets/web/dirbust-heavy.yml similarity index 100% rename from bbot/presets/web_advanced/dirbust-heavy.yml rename to bbot/presets/web/dirbust-heavy.yml diff --git a/bbot/presets/web_advanced/dirbust-light.yml b/bbot/presets/web/dirbust-light.yml similarity index 54% rename from bbot/presets/web_advanced/dirbust-light.yml rename to bbot/presets/web/dirbust-light.yml index be6d3fc0d..d088ee24e 100644 --- a/bbot/presets/web_advanced/dirbust-light.yml +++ b/bbot/presets/web/dirbust-light.yml @@ -1,6 +1,6 @@ description: Basic web directory brute-force (surface-level directories only) -flags: +include: - iis-shortnames modules: @@ -8,9 +8,6 @@ modules: config: modules: - iis_shortnames: - # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames - detect_only: False ffuf: # wordlist size = 1000 lines: 1000 diff --git a/bbot/presets/web/iis-shortnames.yml b/bbot/presets/web/iis-shortnames.yml new file mode 100644 index 000000000..8c8bafbb3 --- /dev/null +++ b/bbot/presets/web/iis-shortnames.yml @@ -0,0 +1,10 @@ +description: Recursively enumerate IIS shortnames, using ffuf to guess the remaining characters + +flags: + - iis-shortnames + +config: + modules: + iis_shortnames: + # exploit the vulnerability + detect_only: false diff --git a/bbot/presets/web_advanced/paramminer.yml b/bbot/presets/web/paramminer.yml similarity index 100% rename from bbot/presets/web_advanced/paramminer.yml rename to bbot/presets/web/paramminer.yml From 7aad074b2b6ae776c6811b2b139ab7c344bea04e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 12:46:01 -0400 Subject: [PATCH 075/171] WIP developer docs --- README.md | 242 +++++++++++------------------------- bbot/presets/web-basic.yml | 3 + docs/contribution.md | 214 +------------------------------ docs/dev/dev_environment.md | 40 ++++++ docs/dev/discord_bot.md | 79 ++++++++++++ docs/dev/index.md | 148 +++++++++++----------- docs/dev/module_howto.md | 168 +++++++++++++++++++++++++ docs/dev/presets.md | 1 + docs/index.md | 25 +++- mkdocs.yml | 29 +++-- 10 files changed, 481 insertions(+), 468 deletions(-) create mode 100644 docs/dev/dev_environment.md create mode 100644 docs/dev/discord_bot.md create mode 100644 docs/dev/module_howto.md create mode 100644 docs/dev/presets.md diff --git a/README.md b/README.md index ae88a990b..97aab9d79 100644 --- a/README.md +++ b/README.md @@ -2,33 +2,13 @@ # BEE·bot -### A Recursive Internet Scanner for Hackers. - [![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) -BBOT (Bighuge BLS OSINT Tool) is a recursive internet scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), but designed to be faster, more reliable, and friendlier to pentesters, bug bounty hunters, and developers. - -Special features include: - -- Support for Multiple Targets -- Web Screenshots -- Suite of Offensive Web Modules -- AI-powered Subdomain Mutations -- Native Output to Neo4j (and more) -- Python API + Developer [Documentation](https://www.blacklanternsecurity.com/bbot/) - -https://github.com/blacklanternsecurity/bbot/assets/20261699/742df3fe-5d1f-4aea-83f6-f990657bf695 +https://github.com/blacklanternsecurity/bbot/assets/20261699/e539e89b-92ea-46fa-b893-9cde94eebf81 _A BBOT scan in real-time - visualization with [VivaGraphJS](https://github.com/blacklanternsecurity/bbot-vivagraphjs)_ -## Quick Start Guide - -Below are some short help sections to get you up and running. - -
-Installation ( Pip ) - -Note: BBOT's [PyPi package](https://pypi.org/project/bbot/) requires Linux and Python 3.9+. +## Installation ```bash # stable version @@ -36,81 +16,105 @@ pipx install bbot # bleeding edge (dev branch) pipx install --pip-args '\--pre' bbot - -bbot --help ``` -
- -
-Installation ( Docker ) +_For more installation methods, including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Getting Started](https://www.blacklanternsecurity.com/bbot/)_ -[Docker images](https://hub.docker.com/r/blacklanternsecurity/bbot) are provided, along with helper script `bbot-docker.sh` to persist your scan data. +## What is BBOT? -```bash -# bleeding edge (dev) -docker run -it blacklanternsecurity/bbot --help +### BBOT is... -# stable -docker run -it blacklanternsecurity/bbot:stable --help +## 1) A Subdomain Finder -# helper script -git clone https://github.com/blacklanternsecurity/bbot && cd bbot -./bbot-docker.sh --help +```bash +# find subdomains of evilcorp.com +bbot -t evilcorp.com -p subdomain-enum ``` -
-
-Example Usage +SEE ALSO: Comparison to Other Tools + +BBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/how_it_works/). + +![subdomain-stats-ebay](https://github.com/blacklanternsecurity/bbot/assets/20261699/53e07e9f-50b6-4b70-9e83-297dbfbcb436) -## Example Commands +
-Scan output, logs, etc. are saved to `~/.bbot`. For more detailed examples and explanations, see [Scanning](https://www.blacklanternsecurity.com/bbot/scanning). - -**Subdomains:** +## 2) A Web Spider ```bash -# Perform a full subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -p subdomain-enum +# crawl evilcorp.com, extracting emails and other goodies +bbot -t evilcorp.com -p spider ``` -**Subdomains (passive only):** +## 3) An Email Gatherer ```bash -# Perform a passive-only subdomain enumeration on evilcorp.com -bbot -t evilcorp.com -p subdomain-enum -rf passive +# enumerate evilcorp.com email addresses +bbot -t evilcorp.com -p email-enum ``` -**Subdomains + port scan + web screenshots:** +## 4) A Web Scanner ```bash -# Port-scan every subdomain, screenshot every webpage, output to current directory -bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o . +# run a light web scan against www.evilcorp.com +bbot -t www.evilcorp.com -p web-basic ``` -**Subdomains + basic web scan:** +## 5) ...And Much More ```bash -# A basic web scan includes wappalyzer, robots.txt, and other non-intrusive web modules -bbot -t evilcorp.com -p subdomain-enum web-basic +# everything everywhere all at once +bbot -t evilcorp.com -p kitchen-sink + +# roughly equivalent to: +bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots ``` -**Web spider:** +## 6) It's Also a Python Library -```bash -# Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc. -bbot -t www.evilcorp.com -p spider -c web_spider_distance=2 web_spider_depth=2 +#### Synchronous +```python +from bbot.scanner import Scanner + +scan = Scanner("evilcorp.com", presets=["subdomain-enum"]) +for event in scan.start(): + print(event) ``` -**Everything everywhere all at once:** +#### Asynchronous +```python +from bbot.scanner import Scanner -```bash -# Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei -bbot -t evilcorp.com -p kitchen-sink +async def main(): + scan = Scanner("evilcorp.com", presets=["subdomain-enum"]) + async for event in scan.async_start(): + print(event.json()) + +import asyncio +asyncio.run(main()) ``` - + +
+SEE ALSO: This Nefarious Discord Bot + +A [BBOT Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/discord_bot/) that responds to the `/scan` command: + +![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) + +## Feature Overview + +BBOT (Bighuge BLS OSINT Tool) is a recursive internet scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), but designed to be faster, more reliable, and friendlier to pentesters, bug bounty hunters, and developers. + +Special features include: + +- Support for Multiple Targets +- Web Screenshots +- Suite of Offensive Web Modules +- AI-powered Subdomain Mutations +- Native Output to Neo4j (and more) +- Python API + Developer Documentation ## Targets @@ -134,7 +138,7 @@ For more information, see [Targets](https://www.blacklanternsecurity.com/bbot/sc Similar to Amass or Subfinder, BBOT supports API keys for various third-party services such as SecurityTrails, etc. -The standard way to do this is to enter your API keys in **`~/.config/bbot/secrets.yml`**: +The standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**: ```yaml modules: shodan_dns: @@ -154,41 +158,7 @@ bbot -c modules.virustotal.api_key=dd5f0eee2e4a99b71a939bded450b246 For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/scanning/configuration/) -## BBOT as a Python Library - -BBOT exposes a Python API that allows it to be used for all kinds of fun and nefarious purposes, like a [Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example) that responds to the `/scan` command. - -![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) - -**Synchronous** - -```python -from bbot.scanner import Scanner - -# any number of targets can be specified -scan = Scanner("example.com", "scanme.nmap.org", modules=["nmap", "sslcert"]) -for event in scan.start(): - print(event.json()) -``` - -**Asynchronous** - -```python -from bbot.scanner import Scanner - -async def main(): - scan = Scanner("example.com", "scanme.nmap.org", modules=["nmap", "sslcert"]) - async for event in scan.async_start(): - print(event.json()) - -import asyncio -asyncio.run(main()) -``` - -
- -
-Documentation - Table of Contents +## Documentation - **User Manual** @@ -229,12 +199,9 @@ asyncio.run(main()) - [Word Cloud](https://www.blacklanternsecurity.com/bbot/dev/helpers/wordcloud) -
- -
-Contribution +## Contribution -BBOT is constantly being improved by the community. Every day it grows more powerful! +Some of the best BBOT modules were written by the community. BBOT is being constantly improved; every day it grows more powerful! We welcome contributions. Not just code, but ideas too! If you have an idea for a new feature, please let us know in [Discussions](https://github.com/blacklanternsecurity/bbot/discussions). If you want to get your hands dirty, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). There you can find setup instructions and a simple tutorial on how to write a BBOT module. We also have extensive [Developer Documentation](https://www.blacklanternsecurity.com/bbot/dev/). @@ -246,71 +213,12 @@ Thanks to these amazing people for contributing to BBOT! :heart:

-Special thanks to the following people who made BBOT possible: +Special thanks to: - @TheTechromancer for creating [BBOT](https://github.com/blacklanternsecurity/bbot) -- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) +- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) and [baddns](https://github.com/blacklanternsecurity/baddns) - Steve Micallef (@smicallef) for creating Spiderfoot - @kerrymilan for his Neo4j and Ansible expertise +- @domwhewell-sage for his family of badass code-looting modules - @aconite33 and @amiremami for their ruthless testing - Aleksei Kornev (@alekseiko) for allowing us ownership of the bbot Pypi repository <3 - -
- -## Comparison to Other Tools - -BBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/how_it_works/). - -![subdomain-stats-ebay](https://github.com/blacklanternsecurity/bbot/assets/20261699/53e07e9f-50b6-4b70-9e83-297dbfbcb436) - -## BBOT Modules By Flag -For a full list of modules, including the data types consumed and emitted by each one, see [List of Modules](https://www.blacklanternsecurity.com/bbot/modules/list_of_modules/). - - -| Flag | # Modules | Description | Modules | -|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 80 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | -| passive | 59 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 45 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | -| active | 43 | Makes active connections to target systems | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dockerhub, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer | -| web-thorough | 29 | More advanced web scanning functionality | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | -| aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | -| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | -| cloud-enum | 12 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth | -| slow | 10 | May take a long time to complete | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | -| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | -| email-enum | 7 | Enumerates email addresses | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert | -| deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | -| portscan | 3 | Discovers open ports | internetdb, masscan, nmap | -| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | -| baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | -| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | -| report | 2 | Generates a report at the end of the scan | affiliates, asn | -| social-enum | 2 | Enumerates social media | httpx, social | -| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | -| subdomain-hijack | 1 | Detects hijackable subdomains | baddns | -| web-screenshots | 1 | Takes screenshots of web pages | gowitness | - - -## BBOT Output Modules -BBOT can save its data to TXT, CSV, JSON, and tons of other destinations including [Neo4j](https://www.blacklanternsecurity.com/bbot/scanning/output/#neo4j), [Splunk](https://www.blacklanternsecurity.com/bbot/scanning/output/#splunk), and [Discord](https://www.blacklanternsecurity.com/bbot/scanning/output/#discord-slack-teams). For instructions on how to use these, see [Output Modules](https://www.blacklanternsecurity.com/bbot/scanning/output). - - -| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | -|-----------------|--------|-----------------|-----------------------------------------------------------------------------------------|---------|--------------------------------------------------------------------------------------------------|---------------------------| -| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | -| csv | output | No | Output to CSV | | * | | -| discord | output | No | Message a Discord channel when certain events are encountered | | * | | -| emails | output | No | Output any email addresses found belonging to the target domain | | EMAIL_ADDRESS | | -| http | output | No | Send every event to a custom URL via a web request | | * | | -| human | output | No | Output to text | | * | | -| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | -| neo4j | output | No | Output to Neo4j | | * | | -| python | output | No | Output via Python API | | * | | -| slack | output | No | Message a Slack channel when certain events are encountered | | * | | -| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | -| subdomains | output | No | Output only resolved, in-scope subdomains | | DNS_NAME, DNS_NAME_UNRESOLVED | | -| teams | output | No | Message a Teams channel when certain events are encountered | | * | | -| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | -| websocket | output | No | Output to websockets | | * | | - diff --git a/bbot/presets/web-basic.yml b/bbot/presets/web-basic.yml index 433cb1e3c..166d973e9 100644 --- a/bbot/presets/web-basic.yml +++ b/bbot/presets/web-basic.yml @@ -1,4 +1,7 @@ description: Quick web scan +include: + - iis-shortnames + flags: - web-basic diff --git a/docs/contribution.md b/docs/contribution.md index 58b1b45e8..b291cea68 100644 --- a/docs/contribution.md +++ b/docs/contribution.md @@ -2,214 +2,8 @@ We welcome contributions! If you have an idea for a new module, or are a Python developer who wants to get involved, please fork us or come talk to us on [Discord](https://discord.com/invite/PZqkgxu5SA). -## Setting Up a Dev Environment +To get started devving, see the following links: -### Installation (Poetry) - -[Poetry](https://python-poetry.org/) is the recommended method of installation if you want to dev on BBOT. To set up a dev environment with Poetry, you can follow these steps: - -- Fork [BBOT](https://github.com/blacklanternsecurity/bbot) on GitHub -- Clone your fork and set up a development environment with Poetry: - -```bash -# clone your forked repo and cd into it -git clone git@github.com//bbot.git -cd bbot - -# install poetry -curl -sSL https://install.python-poetry.org | python3 - - -# install pip dependencies -poetry install -# install pre-commit hooks, etc. -poetry run pre-commit install - -# enter virtual environment -poetry shell - -bbot --help -``` - -- Now, any changes you make in the code will be reflected in the `bbot` command. -- After making your changes, run the tests locally to ensure they pass. - -```bash -# auto-format code indentation, etc. -black . - -# run tests -./bbot/test/run_tests.sh -``` - -- Finally, commit and push your changes, and create a pull request to the `dev` branch of the main BBOT repo. - - -## Creating a Module - -Writing a module is easy and requires only a basic understanding of Python. It consists of a few steps: - -1. Create a new `.py` file in `bbot/modules` -1. At the top of the file, import `BaseModule` -1. Declare a class that inherits from `BaseModule` - - the class must have the same name as your file (case-insensitive) -1. Define in `watched_events` what type of data your module will consume -1. Define in `produced_events` what type of data your module will produce -1. Define (via `flags`) whether your module is `active` or `passive`, and whether it's `safe` or `aggressive` -1. **Put your main logic in `.handle_event()`** - -Here is an example of a simple module that performs whois lookups: - -```python title="bbot/modules/whois.py" -from bbot.modules.base import BaseModule - -class whois(BaseModule): - watched_events = ["DNS_NAME"] # watch for DNS_NAME events - produced_events = ["WHOIS"] # we produce WHOIS events - flags = ["passive", "safe"] - meta = {"description": "Query WhoisXMLAPI for WHOIS data"} - options = {"api_key": ""} # module config options - options_desc = {"api_key": "WhoisXMLAPI Key"} - per_domain_only = True # only run once per domain - - base_url = "https://www.whoisxmlapi.com/whoisserver/WhoisService" - - # one-time setup - runs at the beginning of the scan - async def setup(self): - self.api_key = self.config.get("api_key") - if not self.api_key: - # soft-fail if no API key is set - return None, "Must set API key" - - async def handle_event(self, event): - self.hugesuccess(f"Got {event} (event.data: {event.data})") - _, domain = self.helpers.split_domain(event.data) - url = f"{self.base_url}?apiKey={self.api_key}&domainName={domain}&outputFormat=JSON" - self.hugeinfo(f"Visiting {url}") - response = await self.helpers.request(url) - if response is not None: - await self.emit_event(response.json(), "WHOIS", source=event) -``` - -After saving the module, you can run it with `-m`: - -```bash -# run a scan enabling the module in bbot/modules/mymodule.py -bbot -t evilcorp.com -m whois -``` - -### `handle_event()` and `emit_event()` - -The `handle_event()` method is the most important part of the module. By overriding this method, you control what the module does. During a scan, when an [event](./scanning/events.md) from your `watched_events` is encountered (a `DNS_NAME` in this example), `handle_event()` is automatically called with that event as its argument. - -The `emit_event()` method is how modules return data. When you call `emit_event()`, it creates an [event](./scanning/events.md) and outputs it, sending it any modules that are interested in that data type. - -### `setup()` - -A module's `setup()` method is used for performing one-time setup at the start of the scan, like downloading a wordlist or checking to make sure an API key is valid. It needs to return either: - -1. `True` - module setup succeeded -2. `None` - module setup soft-failed (scan will continue but module will be disabled) -3. `False` - module setup hard-failed (scan will abort) - -Optionally, it can also return a reason. Here are some examples: - -```python -async def setup(self): - if not self.config.get("api_key"): - # soft-fail - return None, "No API key specified" - -async def setup(self): - try: - wordlist = self.helpers.wordlist("https://raw.githubusercontent.com/user/wordlist.txt") - except WordlistError as e: - # hard-fail - return False, f"Error downloading wordlist: {e}" - -async def setup(self): - self.timeout = self.config.get("timeout", 5) - # success - return True -``` - -### Module Config Options - -Each module can have its own set of config options. These live in the `options` and `options_desc` attributes on your class. Both are dictionaries; `options` is for defaults and `options_desc` is for descriptions. Here is a typical example: - -```python title="bbot/modules/nmap.py" -class nmap(BaseModule): - # ... - options = { - "top_ports": 100, - "ports": "", - "timing": "T4", - "skip_host_discovery": True, - } - options_desc = { - "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')", - "ports": "Ports to scan", - "timing": "-T<0-5>: Set timing template (higher is faster)", - "skip_host_discovery": "skip host discovery (-Pn)", - } - - async def setup(self): - self.ports = self.config.get("ports", "") - self.timing = self.config.get("timing", "T4") - self.top_ports = self.config.get("top_ports", 100) - self.skip_host_discovery = self.config.get("skip_host_discovery", True) -``` - -Once you've defined these variables, you can pass the options via `-c`: - -```bash -bbot -m nmap -c modules.nmap.top_ports=250 -``` - -... or via the config: - -```yaml title="~/.config/bbot/bbot.yml" -modules: - nmap: - top_ports: 250 -``` - -Inside the module, you access them via `self.config`, e.g.: - -```python -self.config.get("top_ports") -``` - -### Module Dependencies - -BBOT automates module dependencies with **Ansible**. If your module relies on a third-party binary, OS package, or python library, you can specify them in the `deps_*` attributes of your module. - -```python -class MyModule(BaseModule): - ... - deps_apt = ["chromium-browser"] - deps_ansible = [ - { - "name": "install dev tools", - "package": {"name": ["gcc", "git", "make"], "state": "present"}, - "become": True, - "ignore_errors": True, - }, - { - "name": "Download massdns source code", - "git": { - "repo": "https://github.com/blechschmidt/massdns.git", - "dest": "#{BBOT_TEMP}/massdns", - "single_branch": True, - "version": "master", - }, - }, - { - "name": "Build massdns", - "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"}, - }, - { - "name": "Install massdns", - "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"}, - }, - ] -``` +- [Setting up a Dev Environment](./dev/dev_environment.md) +- [How to Write a BBOT Module](./dev/module_howto.md) +- [Discord Bot Example](./dev/discord_bot.md) diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md new file mode 100644 index 000000000..054656150 --- /dev/null +++ b/docs/dev/dev_environment.md @@ -0,0 +1,40 @@ +## Setting Up a Dev Environment + +### Installation (Poetry) + +[Poetry](https://python-poetry.org/) is the recommended method of installation if you want to dev on BBOT. To set up a dev environment with Poetry, you can follow these steps: + +- Fork [BBOT](https://github.com/blacklanternsecurity/bbot) on GitHub +- Clone your fork and set up a development environment with Poetry: + +```bash +# clone your forked repo and cd into it +git clone git@github.com//bbot.git +cd bbot + +# install poetry +curl -sSL https://install.python-poetry.org | python3 - + +# install pip dependencies +poetry install +# install pre-commit hooks, etc. +poetry run pre-commit install + +# enter virtual environment +poetry shell + +bbot --help +``` + +- Now, any changes you make in the code will be reflected in the `bbot` command. +- After making your changes, run the tests locally to ensure they pass. + +```bash +# auto-format code indentation, etc. +black . + +# run tests +./bbot/test/run_tests.sh +``` + +- Finally, commit and push your changes, and create a pull request to the `dev` branch of the main BBOT repo. diff --git a/docs/dev/discord_bot.md b/docs/dev/discord_bot.md new file mode 100644 index 000000000..9c67efb0a --- /dev/null +++ b/docs/dev/discord_bot.md @@ -0,0 +1,79 @@ + +![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) + +Below is a simple Discord bot designed to run BBOT scans. + +```python +import asyncio +import discord +from discord.ext import commands + +from bbot.scanner import Scanner +from bbot.modules import MODULE_LOADER +from bbot.modules.output.discord import Discord + + +class BBOTDiscordBot(commands.Cog): + """ + A simple Discord bot capable of running a BBOT scan. + + To set up: + 1. Go to Discord Developer Portal (https://discord.com/developers) + 2. Create a new application + 3. Create an invite link for the bot, visit the link to invite it to your server + - Your Application --> OAuth2 --> URL Generator + - For Scopes, select "bot"" + - For Bot Permissions, select: + - Read Messages/View Channels + - Send Messages + 4. Turn on "Message Content Intent" + - Your Application --> Bot --> Privileged Gateway Intents --> Message Content Intent + 5. Copy your Discord Bot Token and put it at the top this file + - Your Application --> Bot --> Reset Token + 6. Run this script + + To scan evilcorp.com, you would type: + + /scan evilcorp.com + + Results will be output to the same channel. + """ + def __init__(self): + self.current_scan = None + + @commands.command(name="scan", description="Scan a target with BBOT.") + async def scan(self, ctx, target: str): + if self.current_scan is not None: + self.current_scan.stop() + await ctx.send(f"Starting scan against {target}.") + + # creates scan instance + self.current_scan = Scanner(target, flags="subdomain-enum") + discord_module = Discord(self.current_scan) + + seen = set() + num_events = 0 + # start scan and iterate through results + async for event in self.current_scan.async_start(): + if hash(event) in seen: + continue + seen.add(hash(event)) + await ctx.send(discord_module.format_message(event)) + num_events += 1 + + await ctx.send(f"Finished scan against {target}. {num_events:,} results.") + self.current_scan = None + + +if __name__ == "__main__": + intents = discord.Intents.default() + intents.message_content = True + bot = commands.Bot(command_prefix="/", intents=intents) + + @bot.event + async def on_ready(): + print(f"We have logged in as {bot.user}") + await bot.add_cog(BBOTDiscordBot()) + + bot.run("DISCORD_BOT_TOKEN_HERE") +``` diff --git a/docs/dev/index.md b/docs/dev/index.md index f2a333cb6..526f03ce9 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -1,88 +1,86 @@ # BBOT Developer Reference -BBOT exposes a convenient API that allows you to create, start, and stop scans using Python code. +BBOT exposes a Python API that allows you to create, start, and stop scans. Documented in this section are commonly-used classes and functions within BBOT, along with usage examples. -## Discord Bot Example +## Running a BBOT Scan from Python -![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) +#### Synchronous +```python +from bbot.scanner import Scanner -Below is a simple Discord bot designed to run BBOT scans. +scan = Scanner("evilcorp.com", presets=["subdomain-enum"]) +for event in scan.start(): + print(event) +``` +#### Asynchronous ```python +from bbot.scanner import Scanner + +async def main(): + scan = Scanner("evilcorp.com", presets=["subdomain-enum"]) + async for event in scan.async_start(): + print(event.json()) + import asyncio -import discord -from discord.ext import commands +asyncio.run(main()) +``` -from bbot.scanner import Scanner -from bbot.modules import MODULE_LOADER -from bbot.modules.output.discord import Discord - - -class BBOTDiscordBot(commands.Cog): - """ - A simple Discord bot capable of running a BBOT scan. - - To set up: - 1. Go to Discord Developer Portal (https://discord.com/developers) - 2. Create a new application - 3. Create an invite link for the bot, visit the link to invite it to your server - - Your Application --> OAuth2 --> URL Generator - - For Scopes, select "bot"" - - For Bot Permissions, select: - - Read Messages/View Channels - - Send Messages - 4. Turn on "Message Content Intent" - - Your Application --> Bot --> Privileged Gateway Intents --> Message Content Intent - 5. Copy your Discord Bot Token and put it at the top this file - - Your Application --> Bot --> Reset Token - 6. Run this script - - To scan evilcorp.com, you would type: - - /scan evilcorp.com - - Results will be output to the same channel. - """ - def __init__(self): - self.current_scan = None - - @commands.command(name="scan", description="Scan a target with BBOT.") - async def scan(self, ctx, target: str): - if self.current_scan is not None: - self.current_scan.stop() - await ctx.send(f"Starting scan against {target}.") - - # creates scan instance - self.current_scan = Scanner(target, flags="subdomain-enum") - discord_module = Discord(self.current_scan) - - seen = set() - num_events = 0 - # start scan and iterate through results - async for event in self.current_scan.async_start(): - if hash(event) in seen: - continue - seen.add(hash(event)) - await ctx.send(discord_module.format_message(event)) - num_events += 1 - - await ctx.send(f"Finished scan against {target}. {num_events:,} results.") - self.current_scan = None - - -if __name__ == "__main__": - intents = discord.Intents.default() - intents.message_content = True - bot = commands.Bot(command_prefix="/", intents=intents) - - @bot.event - async def on_ready(): - print(f"We have logged in as {bot.user}") - await bot.add_cog(BBOTDiscordBot()) - - bot.run("DISCORD_BOT_TOKEN_HERE") +For a full listing of `Scanner` attributes and functions, see the [`Scanner` Code Reference](./scanner.md). + +#### Multiple Targets + +You can specify any number of targets: + +```python +# create a scan against multiple targets +scan = Scanner( + "evilcorp.com", + "evilcorp.org", + "evilcorp.ce", + "4.3.2.1", + "1.2.3.4/24", + presets=["subdomain-enum"] +) + +# this is the same as: +targets = ["evilcorp.com", "evilcorp.org", "evilcorp.ce", "4.3.2.1", "1.2.3.4/24"] +scan = Scanner(*targets, presets=["subdomain-enum"]) +``` + +For more details, including which types of targets are valid, see [Targets](../scanning/index.md#targets) + +#### Other Custom Options + +In many cases, using a [Preset](../scanning/presets.md) like `subdomain-enum` is sufficient. However, the `Scanner` is flexible and accepts many other arguments that can override the default functionality. You can specify [`flags`](../index.md#flags), [`modules`](../index.md#modules), [`output_modules`](../output.md), a [`whitelist` or `blacklist`](../scanning/index.md#whitelists-and-blacklists), and custom [`config` options](../scanning/configuration.md): + +```python +# create a scan against multiple targets +scan = Scanner( + # targets + "evilcorp.com", + "4.3.2.1", + # enable these presets + presets=["subdomain-enum"], + # whitelist these hosts + whitelist=["evilcorp.com", "evilcorp.org"], + # blacklist these hosts + blacklist=["prod.evilcorp.com"], + # also enable these individual modules + modules=["nuclei", "ipstack"], + # exclude modules with these flags + exclude_flags=["slow"], + # custom config options + config={ + "modules": { + "nuclei": { + "tags": "apache,nginx" + } + } + } +) ``` -[Next Up: Scanner -->](scanner.md){ .md-button .md-button--primary } +For a list of all the possible scan options, see the [`Presets` Code Reference](./presets.md) diff --git a/docs/dev/module_howto.md b/docs/dev/module_howto.md new file mode 100644 index 000000000..94d8ffe60 --- /dev/null +++ b/docs/dev/module_howto.md @@ -0,0 +1,168 @@ + +Writing a module is easy and requires only a basic understanding of Python. It consists of a few steps: + +1. Create a new `.py` file in `bbot/modules` +1. At the top of the file, import `BaseModule` +1. Declare a class that inherits from `BaseModule` + - the class must have the same name as your file (case-insensitive) +1. Define in `watched_events` what type of data your module will consume +1. Define in `produced_events` what type of data your module will produce +1. Define (via `flags`) whether your module is `active` or `passive`, and whether it's `safe` or `aggressive` +1. **Put your main logic in `.handle_event()`** + +Here is an example of a simple module that performs whois lookups: + +```python title="bbot/modules/whois.py" +from bbot.modules.base import BaseModule + +class whois(BaseModule): + watched_events = ["DNS_NAME"] # watch for DNS_NAME events + produced_events = ["WHOIS"] # we produce WHOIS events + flags = ["passive", "safe"] + meta = {"description": "Query WhoisXMLAPI for WHOIS data"} + options = {"api_key": ""} # module config options + options_desc = {"api_key": "WhoisXMLAPI Key"} + per_domain_only = True # only run once per domain + + base_url = "https://www.whoisxmlapi.com/whoisserver/WhoisService" + + # one-time setup - runs at the beginning of the scan + async def setup(self): + self.api_key = self.config.get("api_key") + if not self.api_key: + # soft-fail if no API key is set + return None, "Must set API key" + + async def handle_event(self, event): + self.hugesuccess(f"Got {event} (event.data: {event.data})") + _, domain = self.helpers.split_domain(event.data) + url = f"{self.base_url}?apiKey={self.api_key}&domainName={domain}&outputFormat=JSON" + self.hugeinfo(f"Visiting {url}") + response = await self.helpers.request(url) + if response is not None: + await self.emit_event(response.json(), "WHOIS", source=event) +``` + +After saving the module, you can run it with `-m`: + +```bash +# run a scan enabling the module in bbot/modules/mymodule.py +bbot -t evilcorp.com -m whois +``` + +### `handle_event()` and `emit_event()` + +The `handle_event()` method is the most important part of the module. By overriding this method, you control what the module does. During a scan, when an [event](./scanning/events.md) from your `watched_events` is encountered (a `DNS_NAME` in this example), `handle_event()` is automatically called with that event as its argument. + +The `emit_event()` method is how modules return data. When you call `emit_event()`, it creates an [event](./scanning/events.md) and outputs it, sending it any modules that are interested in that data type. + +### `setup()` + +A module's `setup()` method is used for performing one-time setup at the start of the scan, like downloading a wordlist or checking to make sure an API key is valid. It needs to return either: + +1. `True` - module setup succeeded +2. `None` - module setup soft-failed (scan will continue but module will be disabled) +3. `False` - module setup hard-failed (scan will abort) + +Optionally, it can also return a reason. Here are some examples: + +```python +async def setup(self): + if not self.config.get("api_key"): + # soft-fail + return None, "No API key specified" + +async def setup(self): + try: + wordlist = self.helpers.wordlist("https://raw.githubusercontent.com/user/wordlist.txt") + except WordlistError as e: + # hard-fail + return False, f"Error downloading wordlist: {e}" + +async def setup(self): + self.timeout = self.config.get("timeout", 5) + # success + return True +``` + +### Module Config Options + +Each module can have its own set of config options. These live in the `options` and `options_desc` attributes on your class. Both are dictionaries; `options` is for defaults and `options_desc` is for descriptions. Here is a typical example: + +```python title="bbot/modules/nmap.py" +class nmap(BaseModule): + # ... + options = { + "top_ports": 100, + "ports": "", + "timing": "T4", + "skip_host_discovery": True, + } + options_desc = { + "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')", + "ports": "Ports to scan", + "timing": "-T<0-5>: Set timing template (higher is faster)", + "skip_host_discovery": "skip host discovery (-Pn)", + } + + async def setup(self): + self.ports = self.config.get("ports", "") + self.timing = self.config.get("timing", "T4") + self.top_ports = self.config.get("top_ports", 100) + self.skip_host_discovery = self.config.get("skip_host_discovery", True) +``` + +Once you've defined these variables, you can pass the options via `-c`: + +```bash +bbot -m nmap -c modules.nmap.top_ports=250 +``` + +... or via the config: + +```yaml title="~/.config/bbot/bbot.yml" +modules: + nmap: + top_ports: 250 +``` + +Inside the module, you access them via `self.config`, e.g.: + +```python +self.config.get("top_ports") +``` + +### Module Dependencies + +BBOT automates module dependencies with **Ansible**. If your module relies on a third-party binary, OS package, or python library, you can specify them in the `deps_*` attributes of your module. + +```python +class MyModule(BaseModule): + ... + deps_apt = ["chromium-browser"] + deps_ansible = [ + { + "name": "install dev tools", + "package": {"name": ["gcc", "git", "make"], "state": "present"}, + "become": True, + "ignore_errors": True, + }, + { + "name": "Download massdns source code", + "git": { + "repo": "https://github.com/blechschmidt/massdns.git", + "dest": "#{BBOT_TEMP}/massdns", + "single_branch": True, + "version": "master", + }, + }, + { + "name": "Build massdns", + "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"}, + }, + { + "name": "Install massdns", + "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"}, + }, + ] +``` diff --git a/docs/dev/presets.md b/docs/dev/presets.md new file mode 100644 index 000000000..7bc7343e0 --- /dev/null +++ b/docs/dev/presets.md @@ -0,0 +1 @@ +::: bbot.scanner.Preset diff --git a/docs/index.md b/docs/index.md index f2f474c45..7d9894126 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,7 +10,7 @@ _A BBOT scan in real-time - visualization with [VivaGraphJS](https://github.com/ Only **Linux** is supported at this time. **Windows** and **macOS** are *not* supported. If you use one of these platforms, consider using [Docker](#Docker). -BBOT offers multiple methods of installation, including **pipx** and **Docker**. If you plan to dev on BBOT, see [Installation (Poetry)](https://www.blacklanternsecurity.com/bbot/contribution/#installation-poetry). +BBOT offers multiple methods of installation, including **pipx** and **Docker**. If you plan to dev on BBOT, see [Installation (Poetry)](./contribution/#installation-poetry). ### [Python (pip / pipx)](https://pypi.org/project/bbot/) @@ -96,7 +96,26 @@ bbot -t evilcorp.com -p kitchen-sink ## API Keys -No API keys are required to run BBOT. However, some modules need them to function. If you have API keys and want to make use of these modules, you can place them either in BBOT's YAML config (`~/.config/bbot/secrets.yml`): +BBOT works just fine without API keys. However, there are certain modules that need them to function. If you have API keys and want to make use of these modules, you can place them either in your preset: + +```yaml title="my_preset.yml" +description: My custom subdomain enum preset + +include: + - subdomain-enum + - cloud-enum + +config: + modules: + shodan_dns: + api_key: deadbeef + virustotal: + api_key: cafebabe +``` + +...in BBOT's global YAML config (`~/.config/bbot/bbot.yml`): + +Note: this will ensure the API keys are used in all scans, regardless of preset. ```yaml title="~/.config/bbot/secrets.yml" modules: @@ -106,7 +125,7 @@ modules: api_key: cafebabe ``` -Or on the command-line: +...or directly on the command-line: ```bash # specify API key with -c diff --git a/mkdocs.yml b/mkdocs.yml index a2d0c68c7..c2fea7968 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,20 +36,23 @@ nav: - Release History: release_history.md - Troubleshooting: troubleshooting.md - Developer Manual: - - How to Write a Module: contribution.md - Development Overview: dev/index.md - - Scanner: dev/scanner.md - - Event: dev/event.md - - Target: dev/target.md - - BaseModule: dev/basemodule.md - - Helpers: - - Overview: dev/helpers/index.md - - Command: dev/helpers/command.md - - DNS: dev/helpers/dns.md - - Interactsh: dev/helpers/interactsh.md - - Miscellaneous: dev/helpers/misc.md - - Web: dev/helpers/web.md - - Word Cloud: dev/helpers/wordcloud.md + - How to Write a BBOT Module: contribution.md + - Discord Bot Example: dev/discord_bot.md + - Code Reference: + - Presets: dev/presets.md + - Scanner: dev/scanner.md + - Event: dev/event.md + - Target: dev/target.md + - BaseModule: dev/basemodule.md + - Helpers: + - Overview: dev/helpers/index.md + - Command: dev/helpers/command.md + - DNS: dev/helpers/dns.md + - Interactsh: dev/helpers/interactsh.md + - Miscellaneous: dev/helpers/misc.md + - Web: dev/helpers/web.md + - Word Cloud: dev/helpers/wordcloud.md theme: name: material From a732efa59d38acc5292d074eb0e121f67495dccf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 17:12:48 -0400 Subject: [PATCH 076/171] steady work on docs --- README.md | 5 +- bbot/scanner/preset/preset.py | 105 +++++++++++++++++++++++++++++----- bbot/scanner/scanner.py | 62 ++++++++++---------- 3 files changed, 127 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 97aab9d79..3db73f098 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ bbot -t evilcorp.com -p subdomain-enum ```
-SEE ALSO: Comparison to Other Tools +SEE: Comparison to Other Subdomain Enumeration Tools BBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/how_it_works/). @@ -60,6 +60,9 @@ bbot -t evilcorp.com -p email-enum ```bash # run a light web scan against www.evilcorp.com bbot -t www.evilcorp.com -p web-basic + +# run a heavy web scan against www.evilcorp.com +bbot -t www.evilcorp.com -p web-thorough ``` ## 5) ...And Much More diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 0244c40b0..4a3fcafca 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -23,23 +23,56 @@ class Preset: + """ + A preset is the central, all-powerful config in BBOT. It contains everything a scan needs to start. + It can specify scan targets, modules, flags, config options like API keys, etc. + + A preset can be loaded from or saved to YAML. BBOT has a number of ready-made presets for common tasks like + subdomain enumeration, web spidering, dirbusting, etc. + + Attributes: + target (Target): Target(s) of scan. + whitelist (Target): Scan whitelist (by default this is the same as `target`). + blacklist (Target): Scan blacklist (this takes ultimate precedence). + strict_scope (bool): If True, subdomains of targets are not considered to be in-scope. + helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. + output_dir (pathlib.Path): Output directory for scan. + scan_name (str): Name of scan. Defaults to random value, e.g. "demonic_jimmy". + name (str): Human-friendly name of preset. Used mainly for logging purposes. + description (str): Description of preset. + modules (set): Combined modules to enable for the scan. Includes scan modules, internal modules, and output modules. + scan_modules (set): Modules to enable for the scan. + output_modules (set): Output modules to enable for the scan. (note: if no output modules are specified, this is not populated until .bake()) + internal_modules (set): Internal modules for the scan. (note: not populated until .bake()) + exclude_modules (set): Modules to exclude from the scan. When set, automatically removes excluded modules. + flags (set): Flags to enable for the scan. When set, automatically enables modules. + require_flags (set): Require modules to have these flags. When set, automatically removes offending modules. + exclude_flags (set): Exclude modules that have any of these flags. When set, automatically removes offending modules. + module_dirs (set): Custom directories from which to load modules (alias to `self.module_loader.module_dirs`). When set, automatically preloads contained modules. + config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `core.config`) + core (BBOTCore): Local copy of BBOTCore object. + verbose (bool): Whether log level is currently set to verbose. When set, updates log level for all BBOT log handlers. + debug (bool): Whether log level is currently set to debug. When set, updates log level for all BBOT log handlers. + silent (bool): Whether logging is currently disabled. When set to True, silences all stderr. + + Examples: + >>> helper = ConfigAwareHelper(config) + >>> ips = helper.dns.resolve("www.evilcorp.com") + """ def __init__( self, *targets, whitelist=None, blacklist=None, + strict_scope=False, modules=None, output_modules=None, exclude_modules=None, flags=None, require_flags=None, exclude_flags=None, - verbose=False, - debug=False, - silent=False, config=None, - strict_scope=False, module_dirs=None, include=None, output_dir=None, @@ -48,9 +81,41 @@ def __init__( description=None, conditions=None, force=False, + verbose=False, + debug=False, + silent=False, _exclude=None, _log=True, ): + """ + Initializes the Preset class. + + Args: + *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports. + whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`. + blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty. + strict_scope (bool, optional): If True, subdomains of targets are not in-scope. + modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list. + output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json. + exclude_modules (list[str], optional): List of modules to exclude from the scan. + require_flags (list[str], optional): Only enable modules if they have these flags. + exclude_flags (list[str], optional): Don't enable modules if they have any of these flags. + module_dirs (list[str], optional): additional directories to load modules from. + config (dict, optional): Additional scan configuration settings. + include (list[str], optional): names or filenames of other presets to include. + output_dir (str or Path, optional): Directory to store scan output. Defaults to BBOT home directory (`~/.bbot`). + scan_name (str, optional): Human-readable name of the scan. If not specified, it will be random, e.g. "demonic_jimmy". + name (str, optional): Human-readable name of the preset. Used mainly for logging. + description (str, optional): Description of the preset. + conditions (list[str], optional): Custom conditions to be executed before scan start. Written in Jinja2. + force (bool, optional): If True, ignore conditional aborts and failed module setups. Just run the scan! + verbose (bool, optional): Set the BBOT logger to verbose mode. + debug (bool, optional): Set the BBOT logger to debug mode. + silent (bool, optional): Silence all stderr (effectively disables the BBOT logger). + _exclude (bool, optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets. + _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc. + """ + # internal variables self._log = _log self.scan = None self._args = None @@ -58,32 +123,40 @@ def __init__( self._helpers = None self._module_loader = None self._yaml_str = "" - self._modules = set() - - self.explicit_scan_modules = set() if modules is None else set(modules) - self.explicit_output_modules = set() if output_modules is None else set(output_modules) - self._exclude_modules = set() self._require_flags = set() self._exclude_flags = set() self._flags = set() - - self.force = force - self._verbose = False self._debug = False self._silent = False + # these are used only for preserving the modules as specified in the original preset + # this is to ensure the preset looks the same when reserialized + self.explicit_scan_modules = set() if modules is None else set(modules) + self.explicit_output_modules = set() if output_modules is None else set(output_modules) + + # whether to force-start the scan (ignoring conditional aborts and failed module setups) + self.force = force + + # scan output directory self.output_dir = output_dir + # name of scan self.scan_name = scan_name + + # name of preset, default blank self.name = name or "" + # preset description, default blank self.description = description or "" + + # custom conditions, evaluated during .bake() self.conditions = [] if conditions is not None: for condition in conditions: self.conditions.append((self.name, condition)) + # keeps track of loaded preset files to prevent infinite circular inclusions self._preset_files_loaded = set() if _exclude is not None: for _filename in _exclude: @@ -93,10 +166,11 @@ def __init__( self.core = CORE.copy() if config is None: config = omegaconf.OmegaConf.create({}) - # merge any custom configs + # merge custom configs if specified by the user self.core.merge_custom(config) # log verbosity + # setting these automatically sets the log level for all log handlers. if verbose: self.verbose = verbose if debug: @@ -138,19 +212,24 @@ def __init__( modules = [modules] if isinstance(output_modules, str): output_modules = [output_modules] + # requirements/exclusions are always loaded first self.add_excluded_modules(exclude_modules if exclude_modules is not None else []) self.add_required_flags(require_flags if require_flags is not None else []) self.add_excluded_flags(exclude_flags if exclude_flags is not None else []) + # then the modules can be enabled self.add_scan_modules(modules if modules is not None else []) self.add_output_modules(output_modules) # add internal modules + # we enable all of them for now + # if disabled via the config, they are removed during .bake() for internal_module, preloaded in self.module_loader.preloaded(type="internal").items(): is_enabled = self.config.get(internal_module, True) is_excluded = internal_module in self.exclude_modules if is_enabled and not is_excluded: self.add_module(internal_module, module_type="internal") + # adding flags automatically enables populates `self.modules` self.add_flags(flags if flags is not None else []) @property diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 33065f377..0d64b19cb 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -67,16 +67,18 @@ class Scanner: - "FINISHED" (8): Status when the scan has successfully completed. ``` _status_code (int): The numerical representation of the current scan status, stored for internal use. It is mapped according to the values in `_status_codes`. - target (Target): Target of scan - config (omegaconf.dictconfig.DictConfig): BBOT config - whitelist (Target): Scan whitelist (by default this is the same as `target`) - blacklist (Target): Scan blacklist (this takes ultimate precedence) - helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. - manager (ScanManager): Coordinates and monitors the flow of events between modules during a scan - dispatcher (Dispatcher): Triggers certain events when the scan `status` changes - modules (dict): Holds all loaded modules in this format: `{"module_name": Module()}` - stats (ScanStats): Holds high-level scan statistics such as how many events have been produced and consumed by each module - home (pathlib.Path): Base output directory of the scan (default: `~/.bbot/scans/`) + target (Target): Target of scan (alias to `self.preset.target`). + config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `self.preset.config`). + whitelist (Target): Scan whitelist (by default this is the same as `target`) (alias to `self.preset.whitelist`). + blacklist (Target): Scan blacklist (this takes ultimate precedence) (alias to `self.preset.blacklist`). + helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. (alias to `self.preset.helpers`). + output_dir (pathlib.Path): Output directory for scan (alias to `self.preset.output_dir`). + name (str): Name of scan (alias to `self.preset.scan_name`). + manager (ScanManager): Coordinates and monitors the flow of events between modules during a scan. + dispatcher (Dispatcher): Triggers certain events when the scan `status` changes. + modules (dict): Holds all loaded modules in this format: `{"module_name": Module()}`. + stats (ScanStats): Holds high-level scan statistics such as how many events have been produced and consumed by each module. + home (pathlib.Path): Base output directory of the scan (default: `~/.bbot/scans/`). running (bool): Whether the scan is currently running. stopping (bool): Whether the scan is currently stopping. stopped (bool): Whether the scan is currently stopped. @@ -102,38 +104,33 @@ class Scanner: def __init__( self, - *args, + *targets, scan_id=None, dispatcher=None, - force_start=False, - **preset_kwargs, + **kwargs, ): """ Initializes the Scanner class. + If a premade `preset` is specified, it will be used for the scan. + Otherwise, `Scan` accepts the same arguments as `Preset`, which are passed through and used to create a new preset. + Args: - *targets (str): Target(s) to scan. - whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`. - blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty. + *targets (list[str], optional): Scan targets (passed through to `Preset`). + preset (Preset, optional): Preset to use for the scan. scan_id (str, optional): Unique identifier for the scan. Auto-generates if None. - name (str, optional): Human-readable name of the scan. Auto-generates if None. - modules (list[str], optional): List of module names to use during the scan. Defaults to empty list. - output_modules (list[str], optional): List of output modules to use. Defaults to ['python']. - output_dir (str or Path, optional): Directory to store scan output. Defaults to BBOT home directory (`~/.bbot`). - config (dict, optional): Configuration settings. Merged with BBOT config. dispatcher (Dispatcher, optional): Dispatcher object to use. Defaults to new Dispatcher. - strict_scope (bool, optional): If True, only targets explicitly in whitelist are scanned. Defaults to False. - force_start (bool, optional): If True, allows the scan to start even when module setups hard-fail. Defaults to False. + *kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`). """ if scan_id is not None: self.id = str(id) else: self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" - preset = preset_kwargs.pop("preset", None) - preset_kwargs["_log"] = True + preset = kwargs.pop("preset", None) + kwargs["_log"] = True if preset is None: - preset = Preset(*args, **preset_kwargs) + preset = Preset(*targets, **kwargs) self.preset = preset.bake() self.preset.scan = self @@ -162,7 +159,6 @@ def __init__( else: self.home = self.preset.bbot_home / "scans" / self.name - self.force_start = force_start self._status = "NOT_STARTED" self._status_code = 0 @@ -435,7 +431,7 @@ async def load_modules(self): 4. Load output modules and updates the `modules` dictionary. 5. Sorts modules based on their `_priority` attribute. - If any modules fail to load or their dependencies fail to install, a ScanError will be raised (unless `self.force_start` is set to True). + If any modules fail to load or their dependencies fail to install, a ScanError will be raised (unless `self.force` is True). Attributes: succeeded, failed (tuple): A tuple containing lists of modules that succeeded or failed during the dependency installation. @@ -443,7 +439,7 @@ async def load_modules(self): failed, failed_internal, failed_output (list): Lists of module names that failed to load. Raises: - ScanError: If any module dependencies fail to install or modules fail to load, and if self.force_start is False. + ScanError: If any module dependencies fail to install or modules fail to load, and if `self.force` is False. Returns: None @@ -677,6 +673,10 @@ def blacklist(self): def helpers(self): return self.preset.helpers + @property + def force(self): + return self.preset.force + @property def word_cloud(self): return self.helpers.word_cloud @@ -941,9 +941,9 @@ def _stop_log_handlers(self): def _fail_setup(self, msg): msg = str(msg) - if not self.force_start: + if not self.force: msg += " (--force to run module anyway)" - if self.force_start: + if self.force: self.error(msg) else: raise ScanError(msg) From 9031e261cef899e4362383b98075516795f0cfc8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 28 Mar 2024 23:39:21 -0400 Subject: [PATCH 077/171] fix tests --- bbot/scanner/preset/conditions.py | 2 +- bbot/scanner/preset/preset.py | 8 ++++---- bbot/scanner/scanner.py | 13 ++++++------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/bbot/scanner/preset/conditions.py b/bbot/scanner/preset/conditions.py index 54978c2b0..3adacfb91 100644 --- a/bbot/scanner/preset/conditions.py +++ b/bbot/scanner/preset/conditions.py @@ -21,7 +21,7 @@ def context(self): } def abort(self, message): - if not self.preset.force: + if not self.preset.force_start: raise PresetAbortError(message) def warn(self, message): diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 4a3fcafca..9a7fdccc7 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -80,7 +80,7 @@ def __init__( name=None, description=None, conditions=None, - force=False, + force_start=False, verbose=False, debug=False, silent=False, @@ -108,7 +108,7 @@ def __init__( name (str, optional): Human-readable name of the preset. Used mainly for logging. description (str, optional): Description of the preset. conditions (list[str], optional): Custom conditions to be executed before scan start. Written in Jinja2. - force (bool, optional): If True, ignore conditional aborts and failed module setups. Just run the scan! + force_start (bool, optional): If True, ignore conditional aborts and failed module setups. Just run the scan! verbose (bool, optional): Set the BBOT logger to verbose mode. debug (bool, optional): Set the BBOT logger to debug mode. silent (bool, optional): Silence all stderr (effectively disables the BBOT logger). @@ -138,7 +138,7 @@ def __init__( self.explicit_output_modules = set() if output_modules is None else set(output_modules) # whether to force-start the scan (ignoring conditional aborts and failed module setups) - self.force = force + self.force_start = force_start # scan output directory self.output_dir = output_dir @@ -281,7 +281,7 @@ def merge(self, other): if other.conditions: self.conditions.extend(other.conditions) # misc - self.force = self.force | other.force + self.force_start = self.force_start | other.force_start def bake(self): """ diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 0d64b19cb..214953801 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -431,7 +431,7 @@ async def load_modules(self): 4. Load output modules and updates the `modules` dictionary. 5. Sorts modules based on their `_priority` attribute. - If any modules fail to load or their dependencies fail to install, a ScanError will be raised (unless `self.force` is True). + If any modules fail to load or their dependencies fail to install, a ScanError will be raised (unless `self.force_start` is True). Attributes: succeeded, failed (tuple): A tuple containing lists of modules that succeeded or failed during the dependency installation. @@ -439,7 +439,7 @@ async def load_modules(self): failed, failed_internal, failed_output (list): Lists of module names that failed to load. Raises: - ScanError: If any module dependencies fail to install or modules fail to load, and if `self.force` is False. + ScanError: If any module dependencies fail to install or modules fail to load, and if `self.force_start` is False. Returns: None @@ -674,8 +674,8 @@ def helpers(self): return self.preset.helpers @property - def force(self): - return self.preset.force + def force_start(self): + return self.preset.force_start @property def word_cloud(self): @@ -941,11 +941,10 @@ def _stop_log_handlers(self): def _fail_setup(self, msg): msg = str(msg) - if not self.force: - msg += " (--force to run module anyway)" - if self.force: + if self.force_start: self.error(msg) else: + msg += " (--force to run module anyway)" raise ScanError(msg) @property From b1c0200f0f6ba5b8a8c3a6f95381abbec573dca5 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 29 Mar 2024 16:52:51 -0400 Subject: [PATCH 078/171] better preset baking, more docs --- README.md | 40 +- bbot/cli.py | 31 +- bbot/core/helpers/depsinstaller/installer.py | 2 +- bbot/core/modules.py | 20 +- bbot/scanner/preset/args.py | 71 +-- bbot/scanner/preset/path.py | 2 +- bbot/scanner/preset/preset.py | 442 +++++++++--------- bbot/scanner/scanner.py | 17 +- bbot/test/bbot_fixtures.py | 3 +- bbot/test/test_step_1/test_cli.py | 16 +- .../test_manager_scope_accuracy.py | 19 +- bbot/test/test_step_1/test_modules_basic.py | 6 +- bbot/test/test_step_1/test_presets.py | 125 ++--- bbot/test/test_step_1/test_python_api.py | 47 ++ 14 files changed, 425 insertions(+), 416 deletions(-) diff --git a/README.md b/README.md index 3db73f098..1a6f79d9d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![bbot_banner](https://user-images.githubusercontent.com/20261699/158000235-6c1ace81-a267-4f8e-90a1-f4c16884ebac.png)](https://github.com/blacklanternsecurity/bbot) -# BEE·bot +#### BBOT /ˈBEE·bot/ (noun): A recursive internet scanner for hackers. [![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) @@ -26,11 +26,43 @@ _For more installation methods, including [Docker](https://hub.docker.com/r/blac ## 1) A Subdomain Finder +Passive API sources plus a recursive DNS brute-force with target-specific subdomain mutations. + ```bash # find subdomains of evilcorp.com bbot -t evilcorp.com -p subdomain-enum ``` +
+subdomain-enum.yml + +```yaml +description: Enumerate subdomains via APIs, brute-force + +flags: + - subdomain-enum + +output_modules: + - subdomains + +config: + modules: + stdout: + format: text + # only output DNS_NAMEs to the console + event_types: + - DNS_NAME + # only show in-scope subdomains + in_scope_only: True + # display the raw subdomains, nothing else + event_fields: + - data + # automatically dedupe + accept_dups: False +``` + +
+
SEE: Comparison to Other Subdomain Enumeration Tools @@ -52,7 +84,7 @@ bbot -t evilcorp.com -p spider ```bash # enumerate evilcorp.com email addresses -bbot -t evilcorp.com -p email-enum +bbot -t evilcorp.com -p subdomain-enum spider email-enum ``` ## 4) A Web Scanner @@ -100,9 +132,9 @@ asyncio.run(main()) ```
-SEE ALSO: This Nefarious Discord Bot +SEE: This Nefarious Discord Bot -A [BBOT Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/discord_bot/) that responds to the `/scan` command: +A [BBOT Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/discord_bot/) that responds to the `/scan` command. Scan the internet from the comfort of your discord server! ![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) diff --git a/bbot/cli.py b/bbot/cli.py index 0f77615d5..4fc84aebf 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -64,26 +64,23 @@ async def _main(): log.stdout(row) return + # if we're listing modules or their options if options.list_modules or options.list_module_options: - modules_to_list = set() - if options.modules or options.flags: - modules_to_list.update(set(preset.scan_modules)) - if options.output_modules: - modules_to_list.update(set(preset.output_modules)) - + # if no modules or flags are specified, enable everything if not (options.modules or options.output_modules or options.flags): for module, preloaded in preset.module_loader.preloaded().items(): module_type = preloaded.get("type", "scan") preset.add_module(module, module_type=module_type) - modules_to_list.update(set(preset.modules)) + + preset.bake() # --list-modules if options.list_modules: log.stdout("") log.stdout("### MODULES ###") log.stdout("") - for row in preset.module_loader.modules_table(modules_to_list).splitlines(): + for row in preset.module_loader.modules_table(preset.modules).splitlines(): log.stdout(row) return @@ -92,7 +89,7 @@ async def _main(): log.stdout("") log.stdout("### MODULE OPTIONS ###") log.stdout("") - for row in preset.module_loader.modules_options_table(modules_to_list).splitlines(): + for row in preset.module_loader.modules_options_table(preset.modules).splitlines(): log.stdout(row) return @@ -106,19 +103,21 @@ async def _main(): log.stdout(row) return - deadly_modules = [m for m in preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", [])] + try: + scan = Scanner(preset=preset) + except (PresetAbortError, ValidationError) as e: + log.warning(str(e)) + return + + deadly_modules = [ + m for m in scan.preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", []) + ] if deadly_modules and not options.allow_deadly: log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") log.hugewarning(f"Deadly modules are highly intrusive") log.hugewarning(f"Please specify --allow-deadly to continue") return False - try: - scan = Scanner(preset=preset) - except PresetAbortError as e: - log.warning(str(e)) - return - # --current-preset if options.current_preset: print(scan.preset.to_yaml()) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 62a627033..8b9d2ae2b 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -59,7 +59,7 @@ async def install(self, *modules): notified = False for m in modules: # assume success if we're ignoring dependencies - if self.deps_behavior == "no_deps": + if self.deps_behavior == "disable": succeeded.append(m) continue # abort if module name is unknown diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 4f2824229..62929230c 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -46,7 +46,11 @@ def __init__(self): self.__preloaded = {} self._modules = {} self._configs = {} - self._all_flags = set() + self.flag_choices = set() + self.all_module_choices = set() + self.scan_module_choices = set() + self.output_module_choices = set() + self.internal_module_choices = set() self._preload_cache = None @@ -147,8 +151,8 @@ def preload(self, module_dirs=None): module_type = str(module_dir.name) elif module_dir.name not in ("modules"): flags = set(preloaded["flags"] + [module_dir.name]) - self._all_flags.update(flags) preloaded["flags"] = sorted(flags) + # derive module dependencies from watched event types (only for scan modules) if module_type == "scan": for event_type in preloaded["watched_events"]: @@ -156,6 +160,7 @@ def preload(self, module_dirs=None): deps_modules = set(preloaded.get("deps", {}).get("modules", [])) deps_modules.add(self.default_module_deps[event_type]) preloaded["deps"]["modules"] = sorted(deps_modules) + preloaded["type"] = module_type preloaded["namespace"] = namespace preloaded["cache_key"] = module_cache_key @@ -165,8 +170,17 @@ def preload(self, module_dirs=None): log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") sys.exit(1) + self.all_module_choices.add(module_name) + module_type = preloaded.get("type", "scan") + if module_type == "scan": + self.scan_module_choices.add(module_name) + elif module_type == "output": + self.output_module_choices.add(module_name) + elif module_type == "internal": + self.internal_module_choices.add(module_name) + flags = preloaded.get("flags", []) - self._all_flags.update(set(flags)) + self.flag_choices.update(set(flags)) self.__preloaded[module_name] = preloaded config = OmegaConf.create(preloaded.get("config", {})) diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 73bc63773..943b693cd 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -74,15 +74,6 @@ def __init__(self, preset): self.preset = preset self._config = None - # validate module choices - self._module_choices = sorted(set(self.preset.module_loader.preloaded(type="scan"))) - self._output_module_choices = sorted(set(self.preset.module_loader.preloaded(type="output"))) - self._all_module_choices = sorted(set(self.preset.module_loader.preloaded())) - self._flag_choices = set() - for m, c in self.preset.module_loader.preloaded().items(): - self._flag_choices.update(set(c.get("flags", []))) - self._flag_choices = sorted(self._flag_choices) - self.parser = self.create_parser() self._parsed = None @@ -120,21 +111,15 @@ def preset_from_args(self): except BBOTArgumentError: raise except Exception as e: - raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e} (-d to debug)') - - # then we validate the modules/flags/config options - self.validate() - - # load modules & flags (excluded then required then all others) - args_preset.add_excluded_modules(self.parsed.exclude_modules) - args_preset.add_excluded_flags(self.parsed.exclude_flags) - args_preset.add_required_flags(self.parsed.require_flags) - for scan_module in self.parsed.modules: - args_preset.add_module(scan_module, module_type="scan") - for output_module in self.parsed.output_modules: - args_preset.add_module(output_module, module_type="output") + raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}') - args_preset.add_flags(self.parsed.flags) + # modules + flags + args_preset.exclude_modules.update(set(self.parsed.exclude_modules)) + args_preset.exclude_flags.update(set(self.parsed.exclude_flags)) + args_preset.require_flags.update(set(self.parsed.require_flags)) + args_preset.explicit_scan_modules.update(set(self.parsed.modules)) + args_preset.explicit_output_modules.update(set(self.parsed.output_modules)) + args_preset.flags.update(set(self.parsed.flags)) # dependencies if self.parsed.retry_deps: @@ -149,7 +134,7 @@ def preset_from_args(self): # other scan options args_preset.scan_name = self.parsed.name args_preset.output_dir = self.parsed.output_dir - args_preset.force = self.parsed.force + args_preset.force_start = self.parsed.force # CLI config options (dot-syntax) for config_arg in self.parsed.config: @@ -209,7 +194,7 @@ def create_parser(self, *args, **kwargs): "--modules", nargs="+", default=[], - help=f'Modules to enable. Choices: {",".join(self._module_choices)}', + help=f'Modules to enable. Choices: {",".join(self.preset.module_loader.scan_module_choices)}', metavar="MODULE", ) modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") @@ -219,12 +204,20 @@ def create_parser(self, *args, **kwargs): modules.add_argument( "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" ) + modules.add_argument( + "-om", + "--output-modules", + nargs="+", + default=[], + help=f'Output module(s). Choices: {",".join(self.preset.module_loader.output_module_choices)}', + metavar="MODULE", + ) modules.add_argument( "-f", "--flags", nargs="+", default=[], - help=f'Enable modules by flag. Choices: {",".join(self._flag_choices)}', + help=f'Enable modules by flag. Choices: {",".join(self.preset.module_loader.flag_choices)}', metavar="FLAG", ) modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") @@ -244,14 +237,6 @@ def create_parser(self, *args, **kwargs): help=f"Disable modules with these flags. (e.g. -ef aggressive)", metavar="FLAG", ) - modules.add_argument( - "-om", - "--output-modules", - nargs="+", - default=[], - help=f'Output module(s). Choices: {",".join(self._output_module_choices)}', - metavar="MODULE", - ) modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") scan = p.add_argument_group(title="Scan") scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") @@ -331,20 +316,4 @@ def validate(self): if self.exclude_from_validation.match(c): continue # otherwise, ensure it exists as a module option - raise BBOTArgumentError(get_closest_match(c, all_options, msg="module option")) - - # validate modules - for m in self.parsed.modules: - if m not in self._module_choices: - raise BBOTArgumentError(get_closest_match(m, self._module_choices, msg="scan module")) - for m in self.parsed.exclude_modules: - if m not in self._all_module_choices: - raise BBOTArgumentError(get_closest_match(m, self._all_module_choices, msg="module")) - for m in self.parsed.output_modules: - if m not in self._output_module_choices: - raise BBOTArgumentError(get_closest_match(m, self._output_module_choices, msg="output module")) - - # validate flags - for f in set(self.parsed.flags + self.parsed.require_flags + self.parsed.exclude_flags): - if f not in self._flag_choices: - raise BBOTArgumentError(get_closest_match(f, self._flag_choices, msg="flag")) + raise ValidationError(get_closest_match(c, all_options, msg="module option")) diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index 00de89569..469c2dd39 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -40,7 +40,7 @@ def find(self, filename): log.verbose(f'Found preset matching "{filename}" at {file}') self.add_path(file.parent) return file.resolve() - raise BBOTError( + raise ValidationError( f'Could not find preset at "{filename}" - file does not exist. Use -lp to list available presets' ) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9a7fdccc7..d8c068f29 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -12,7 +12,7 @@ from bbot.core import CORE from bbot.core.errors import * from bbot.core.event.base import make_event -from bbot.core.helpers.misc import make_table, mkdir +from bbot.core.helpers.misc import make_table, mkdir, get_closest_match log = logging.getLogger("bbot.presets") @@ -24,12 +24,23 @@ class Preset: """ - A preset is the central, all-powerful config in BBOT. It contains everything a scan needs to start. - It can specify scan targets, modules, flags, config options like API keys, etc. + A preset is the central config for a BBOT scan. It contains everything a scan needs to run -- + targets, modules, flags, config options like API keys, etc. - A preset can be loaded from or saved to YAML. BBOT has a number of ready-made presets for common tasks like + You can create a preset manually and pass it into `Scanner(preset=preset)`. + Or, you can pass `Preset`'s kwargs into `Scanner()` and it will create the preset for you implicitly. + + Presets can include other presets (which can in turn include other presets, and so on). + This works by merging each preset in turn using `Preset.merge()`. + The order matters. In case of a conflict, the last preset to be merged wins priority. + + Presets can be loaded from or saved to YAML. BBOT has a number of ready-made presets for common tasks like subdomain enumeration, web spidering, dirbusting, etc. + Presets are highly customizable via `conditions`, which use the Jinja2 templating engine. + Using `conditions`, you can define custom logic to inspect the final preset before the scan starts, and change it if need be. + Based on the state of the preset, you can print a warning message, abort the scan, enable/disable modules, etc.. + Attributes: target (Target): Target(s) of scan. whitelist (Target): Scan whitelist (by default this is the same as `target`). @@ -56,8 +67,17 @@ class Preset: silent (bool): Whether logging is currently disabled. When set to True, silences all stderr. Examples: - >>> helper = ConfigAwareHelper(config) - >>> ips = helper.dns.resolve("www.evilcorp.com") + >>> preset = Preset( + "evilcorp.com", + "1.2.3.0/24", + flags=["subdomain-enum"], + modules=["nuclei"], + config={"http_proxy": "http://127.0.0.1"} + ) + >>> scan = Scanner(preset=preset) + + >>> preset = Preset.from_yaml_file("my_preset.yml") + >>> scan = Scanner(preset=preset) """ def __init__( @@ -116,6 +136,7 @@ def __init__( _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc. """ # internal variables + self._cli = False self._log = _log self.scan = None self._args = None @@ -123,15 +144,43 @@ def __init__( self._helpers = None self._module_loader = None self._yaml_str = "" - self._modules = set() - self._exclude_modules = set() - self._require_flags = set() - self._exclude_flags = set() - self._flags = set() self._verbose = False self._debug = False self._silent = False + # modules / flags + self.modules = set() + self.exclude_modules = set() + self.flags = set() + self.exclude_flags = set() + self.require_flags = set() + + # modules + flags + if modules is None: + modules = [] + if isinstance(modules, str): + modules = [modules] + if output_modules is None: + output_modules = [] + if isinstance(output_modules, str): + output_modules = [output_modules] + if exclude_modules is None: + exclude_modules = [] + if isinstance(exclude_modules, str): + exclude_modules = [exclude_modules] + if flags is None: + flags = [] + if isinstance(flags, str): + flags = [flags] + if exclude_flags is None: + exclude_flags = [] + if isinstance(exclude_flags, str): + exclude_flags = [exclude_flags] + if require_flags is None: + require_flags = [] + if isinstance(require_flags, str): + require_flags = [require_flags] + # these are used only for preserving the modules as specified in the original preset # this is to ensure the preset looks the same when reserialized self.explicit_scan_modules = set() if modules is None else set(modules) @@ -203,34 +252,13 @@ def __init__( for included_preset in include: self.include_preset(included_preset) - # modules + flags - if modules is None: - modules = [] - if output_modules is None: - output_modules = [] - if isinstance(modules, str): - modules = [modules] - if isinstance(output_modules, str): - output_modules = [output_modules] - # requirements/exclusions are always loaded first - self.add_excluded_modules(exclude_modules if exclude_modules is not None else []) - self.add_required_flags(require_flags if require_flags is not None else []) - self.add_excluded_flags(exclude_flags if exclude_flags is not None else []) - # then the modules can be enabled - self.add_scan_modules(modules if modules is not None else []) - self.add_output_modules(output_modules) - - # add internal modules - # we enable all of them for now - # if disabled via the config, they are removed during .bake() - for internal_module, preloaded in self.module_loader.preloaded(type="internal").items(): - is_enabled = self.config.get(internal_module, True) - is_excluded = internal_module in self.exclude_modules - if is_enabled and not is_excluded: - self.add_module(internal_module, module_type="internal") - - # adding flags automatically enables populates `self.modules` - self.add_flags(flags if flags is not None else []) + # we don't fill self.modules yet (that happens in .bake()) + self.explicit_scan_modules.update(set(modules)) + self.explicit_output_modules.update(set(output_modules)) + self.exclude_modules.update(set(exclude_modules)) + self.flags.update(set(flags)) + self.exclude_flags.update(set(exclude_flags)) + self.require_flags.update(set(require_flags)) @property def bbot_home(self): @@ -241,6 +269,25 @@ def preset_dir(self): return self.bbot_home / "presets" def merge(self, other): + """ + Merge another preset into this one. + + If there are any config conflicts, `other` will win over `self`. + + Args: + other (Preset): The preset to merge into this one. + + Example: + >>> preset1 = Preset(modules=["nmap"]) + >>> preset1.scan_modules + ['nmap'] + >>> preset2 = Preset(modules=["sslcert"]) + >>> preset2.scan_modules + ['sslcert'] + >>> preset1.merge(preset2) + >>> preset1.scan_modules + ['nmap', 'sslcert'] + """ self.log_debug(f'Merging preset "{other.name}" into "{self.name}"') # config self.core.merge_custom(other.core.custom_config) @@ -248,16 +295,13 @@ def merge(self, other): # module dirs # modules + flags # establish requirements / exclusions first - self.add_excluded_modules(other.exclude_modules) - self.add_required_flags(other.require_flags) - self.add_excluded_flags(other.exclude_flags) + self.exclude_modules.update(other.exclude_modules) + self.require_flags.update(other.require_flags) + self.exclude_flags.update(other.exclude_flags) # then it's okay to start enabling modules self.explicit_scan_modules.update(other.explicit_scan_modules) self.explicit_output_modules.update(other.explicit_output_modules) - self.add_flags(other.flags) - for module_name in other.modules: - module_type = self.preloaded_module(module_name).get("type", "scan") - self.add_module(module_name, module_type=module_type) + self.flags.update(other.flags) # scope self.target.add_target(other.target) self.whitelist.add_target(other.whitelist) @@ -285,8 +329,15 @@ def merge(self, other): def bake(self): """ - return a "baked" copy of the preset, ready for use by a BBOT scan + Return a "baked" copy of this preset, ready for use by a BBOT scan. + + Baking a preset finalizes it by populating `preset.modules` based on flags, + performing final validations, and substituting environment variables in preloaded modules. + It also evaluates custom `conditions` as specified in the preset. + + This function is automatically called in Scanner.__init__(). There is no need to call it manually. """ + self.log_debug("Getting baked") # create a copy of self baked_preset = copy(self) # copy core @@ -303,15 +354,40 @@ def bake(self): os.environ.clear() os.environ.update(os_environ) + # validate flags, config options + baked_preset.validate() + + # now that our requirements / exclusions are validated, we can start enabling modules + # enable scan modules + for module in baked_preset.explicit_scan_modules: + baked_preset.add_module(module, module_type="scan") + # enable output modules + for module in self.explicit_output_modules: + baked_preset.add_module(module, module_type="output", raise_error=False) + + # enable internal modules + for internal_module, preloaded in baked_preset.module_loader.preloaded(type="internal").items(): + is_enabled = baked_preset.config.get(internal_module, True) + is_excluded = internal_module in baked_preset.exclude_modules + if is_enabled and not is_excluded: + baked_preset.add_module(internal_module, module_type="internal", raise_error=False) + # disable internal modules if requested for internal_module in baked_preset.internal_modules: if baked_preset.config.get(internal_module, True) == False: - baked_preset.exclude_module(internal_module) + baked_preset.exclude_modules.add(internal_module) + + # enable modules by flag + for flag in baked_preset.flags: + for module, preloaded in baked_preset.module_loader.preloaded().items(): + module_flags = preloaded.get("flags", []) + if flag in module_flags: + baked_preset.add_module(module, raise_error=False) # ensure we have output modules - if not self.output_modules: + if not baked_preset.output_modules: for output_module in ("python", "csv", "human", "json"): - self.add_module(output_module, module_type="output") + baked_preset.add_module(output_module, module_type="output", raise_error=False) # evaluate conditions if baked_preset.conditions: @@ -323,6 +399,12 @@ def bake(self): return baked_preset def parse_args(self): + """ + Parse CLI arguments, and merge them into this preset. + + Used in `cli.py`. + """ + self._cli = True self.merge(self.args.preset_from_args()) @property @@ -338,205 +420,29 @@ def module_dirs(self, module_dirs): self.module_loader.add_module_dir(m) self._module_dirs.add(m) - @property - def modules(self): - return self._modules - - @modules.setter - def modules(self, modules): - if isinstance(modules, str): - modules = [modules] - modules = set(modules) - modules.update(self.internal_modules) - for module_name in modules: - self.add_module(module_name) - @property def scan_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "scan"] - @scan_modules.setter - def scan_modules(self, modules): - self.log_debug(f"Setting scan modules to {modules}") - self._modules_setter(modules, module_type="scan") - @property def output_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "output"] - @output_modules.setter - def output_modules(self, modules): - self.log_debug(f"Setting output modules to {modules}") - self._modules_setter(modules, module_type="output") - @property def internal_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] - def _modules_setter(self, modules, module_type="scan"): - if isinstance(modules, str): - modules = [modules] - # start by removing currently-enabled modules of that type - for module_name in list(self.modules): - if module_type and self.preloaded_module(module_name).get("type", "scan") == module_type: - self._modules.remove(module_name) - for module_name in set(modules): - self.add_module(module_name, module_type=module_type) - - def add_scan_modules(self, modules): - for module in modules: - self.add_module(module, module_type="scan") - - def add_output_modules(self, modules): - for module in modules: - self.add_module(module, module_type="output") - - def add_internal_modules(self, modules): - for module in modules: - self.add_module(module, module_type="internal") - - def add_module(self, module_name, module_type="scan"): - if module_name in self.exclude_modules: - self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') - return - if module_name in self.modules: - self.log_debug(f'Already added module "{module_name}"') - return - try: - preloaded = self.module_loader.preloaded()[module_name] - except KeyError: - raise EnableModuleError(f'Unable to add unknown BBOT module "{module_name}"') - - module_flags = preloaded.get("flags", []) - _module_type = preloaded.get("type", "scan") - if module_type: - if _module_type != module_type: - self.log_verbose( - f'Not adding module "{module_name}" because its type ({_module_type}) is not "{module_type}"' - ) - return - - if _module_type == "scan": - for f in module_flags: - if f in self.exclude_flags: - self.log_verbose(f'Skipping module "{module_name}" because it\'s excluded') - return - if self.require_flags and not all(f in module_flags for f in self.require_flags): - self.log_verbose( - f'Skipping module "{module_name}" because it doesn\'t have the required flags ({",".join(self.require_flags)})' - ) - return - + def add_module(self, module_name, module_type="scan", raise_error=True): self.log_debug(f'Adding module "{module_name}"') - self._modules.add(module_name) + is_valid, reason, preloaded = self._is_valid_module(module_name, module_type, raise_error=raise_error) + if not is_valid: + self.log_debug(f'Unable to add {module_type} module "{module_name}": {reason}') + return + self.modules.add(module_name) for module_dep in preloaded.get("deps", {}).get("modules", []): if module_dep != module_name and module_dep not in self.modules: self.log_verbose(f'Adding module "{module_dep}" because {module_name} depends on it') - self.add_module(module_dep) - - @property - def exclude_modules(self): - return self._exclude_modules - - @property - def exclude_flags(self): - return self._exclude_flags - - @property - def require_flags(self): - return self._require_flags - - @property - def flags(self): - return self._flags - - @flags.setter - def flags(self, flags): - self.log_debug(f"Setting flags to {flags}") - flags = set(flags) - self._flags = flags - self.add_flags(flags) - - def add_flags(self, flags): - if flags: - self.log_debug(f"Adding flags: {flags}") - for flag in flags: - self.add_flag(flag) - - def add_flag(self, flag): - if not flag in self.module_loader._all_flags: - raise EnableFlagError(f'Flag "{flag}" was not found') - if flag in self.exclude_flags: - self.log_debug(f'Skipping flag "{flag}" because it\'s excluded') - return - self.log_debug(f'Adding flag "{flag}"') - self._flags.add(flag) - for module, preloaded in self.module_loader.preloaded().items(): - module_flags = preloaded.get("flags", []) - if flag in module_flags: - self.add_module(module) - - @require_flags.setter - def require_flags(self, flags): - self.log_debug(f"Setting required flags to {flags}") - if isinstance(flags, str): - flags = [flags] - self._require_flags = set() - for flag in set(flags): - self.require_flag(flag) - - @exclude_modules.setter - def exclude_modules(self, modules): - self.log_debug(f"Setting excluded modules to {modules}") - if isinstance(modules, str): - modules = [modules] - self._exclude_modules = set() - for module in set(modules): - self.exclude_module(module) - - @exclude_flags.setter - def exclude_flags(self, flags): - self.log_debug(f"Setting excluded flags to {flags}") - if isinstance(flags, str): - flags = [flags] - self._exclude_flags = set() - for flag in set(flags): - self.exclude_flag(flag) - - def add_required_flags(self, flags): - for flag in flags: - self.require_flag(flag) - - def require_flag(self, flag): - self.require_flags.add(flag) - for module in list(self.scan_modules): - module_flags = self.preloaded_module(module).get("flags", []) - if flag not in module_flags: - self.log_verbose(f'Removing module "{module}" because it doesn\'t have the required flag, "{flag}"') - self.modules.remove(module) - - def add_excluded_flags(self, flags): - for flag in flags: - self.exclude_flag(flag) - - def exclude_flag(self, flag): - self.exclude_flags.add(flag) - for module in list(self.scan_modules): - module_flags = self.preloaded_module(module).get("flags", []) - if flag in module_flags: - self.log_verbose(f'Removing module "{module}" because it has the excluded flag, "{flag}"') - self.modules.remove(module) - - def add_excluded_modules(self, modules): - for module in modules: - self.exclude_module(module) - - def exclude_module(self, module): - self.exclude_modules.add(module) - for module in list(self.modules): - if module in self.exclude_modules: - self.log_verbose(f'Removing module "{module}" because it\'s excluded') - self.modules.remove(module) + self.add_module(module_dep, raise_error=False) def preloaded_module(self, module): return self.module_loader.preloaded()[module] @@ -790,6 +696,75 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) + def _is_valid_module(self, module, module_type, name_only=False, raise_error=True): + if module_type == "scan": + module_choices = self.module_loader.scan_module_choices + elif module_type == "output": + module_choices = self.module_loader.output_module_choices + elif module_type == "internal": + module_choices = self.module_loader.internal_module_choices + else: + raise ValidationError(f'Unknown module type "{module}"') + + if not module in module_choices: + raise ValidationError(get_closest_match(module, module_choices, msg=f"{module_type} module")) + + try: + preloaded = self.module_loader.preloaded()[module] + except KeyError: + raise ValidationError(f'Unknown module "{module}"') + + if name_only: + return True, "", preloaded + + if module in self.exclude_modules: + reason = "the module has been excluded" + if raise_error: + raise ValidationError(f'Unable to add {module_type} module "{module}" because {reason}') + return False, reason, {} + + module_flags = preloaded.get("flags", []) + _module_type = preloaded.get("type", "scan") + if module_type: + if _module_type != module_type: + reason = f'its type ({_module_type}) is not "{module_type}"' + if raise_error: + raise ValidationError(f'Unable to add {module_type} module "{module}" because {reason}') + return False, reason, preloaded + + if _module_type == "scan": + if self.exclude_flags: + for f in module_flags: + if f in self.exclude_flags: + return False, f'it has excluded flag, "{f}"', preloaded + if self.require_flags and not all(f in module_flags for f in self.require_flags): + return False, f'it doesn\'t have the required flags ({",".join(self.require_flags)})', preloaded + + return True, "", preloaded + + def validate(self): + if self._cli: + self.args.validate() + + # validate excluded modules + for excluded_module in self.exclude_modules: + if not excluded_module in self.module_loader.all_module_choices: + raise ValidationError( + get_closest_match(excluded_module, self.module_loader.all_module_choices, msg="module") + ) + # validate excluded flags + for excluded_flag in self.exclude_flags: + if not excluded_flag in self.module_loader.flag_choices: + raise ValidationError(get_closest_match(excluded_flag, self.module_loader.flag_choices, msg="flag")) + # validate required flags + for required_flag in self.require_flags: + if not required_flag in self.module_loader.flag_choices: + raise ValidationError(get_closest_match(required_flag, self.module_loader.flag_choices, msg="flag")) + # validate flags + for flag in self.flags: + if not flag in self.module_loader.flag_choices: + raise ValidationError(get_closest_match(flag, self.module_loader.flag_choices, msg="flag")) + @property def all_presets(self): preset_dir = self.preset_dir @@ -852,6 +827,7 @@ def presets_table(self, include_modules=True): if include_modules: header.append("Modules") for yaml_file, (loaded_preset, category, preset_path, original_file) in self.all_presets.items(): + loaded_preset = loaded_preset.bake() num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] if include_modules: diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 214953801..c859d6c42 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -20,10 +20,10 @@ from .manager import ScanManager from .dispatcher import Dispatcher from bbot.core.event import make_event -from bbot.core.errors import BBOTError, ScanError from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen +from bbot.core.errors import BBOTError, ScanError, ValidationError log = logging.getLogger("bbot.scanner") @@ -131,6 +131,9 @@ def __init__( kwargs["_log"] = True if preset is None: preset = Preset(*targets, **kwargs) + else: + if not isinstance(preset, Preset): + raise ValidationError(f'Preset must be of type Preset, not "{type(preset).__name__}"') self.preset = preset.bake() self.preset.scan = self @@ -226,11 +229,16 @@ def __init__( async def _prep(self): """ - Calls .load_modules() and .setup_modules() in preparation for a scan + Creates the scan's output folder, loads its modules, and calls their .setup() methods. """ self.helpers.mkdir(self.home) if not self._prepped: + # save scan preset + with open(self.home / "preset.yml", "w") as f: + f.write(self.preset.to_yaml()) + + # log scan overview start_msg = f"Scan with {len(self.preset.scan_modules):,} modules seeded with {len(self.target):,} targets" details = [] if self.whitelist != self.target: @@ -241,14 +249,19 @@ async def _prep(self): start_msg += f" ({', '.join(details)})" self.hugeinfo(start_msg) + # load scan modules (this imports and instantiates them) + # up to this point they were only preloaded await self.load_modules() + # run each module's .setup() method self.info(f"Setting up modules...") succeeded, hard_failed, soft_failed = await self.setup_modules() + # abort if there are no output modules num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) if num_output_modules < 1: raise ScanError("Failed to load output modules. Aborting.") + # abort if any of the module .setup()s hard-failed (i.e. they errored or returned False) total_failed = len(hard_failed + soft_failed) if hard_failed: msg = f"Setup hard-failed for {len(hard_failed):,} modules ({','.join(hard_failed)})" diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 5185e83d0..d34cdf730 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -13,6 +13,7 @@ from werkzeug.wrappers import Request from bbot.core import CORE +from bbot.core.errors import * # noqa: F401 from bbot.scanner import Preset from bbot.core.helpers.misc import mkdir @@ -50,8 +51,6 @@ def match_data(self, request: Request) -> bool: pytest_httpserver.httpserver.RequestMatcher = SubstringRequestMatcher -from bbot.core.errors import * # noqa: F401 - # silence pytest_httpserver log = logging.getLogger("werkzeug") log.setLevel(logging.CRITICAL) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 8343e783b..dce77371c 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -20,10 +20,16 @@ async def test_cli_scan(monkeypatch): assert result == True scan_home = scans_home / "test_cli_scan" + assert (scan_home / "preset.yml").is_file(), "preset.yml not found" assert (scan_home / "wordcloud.tsv").is_file(), "wordcloud.tsv not found" assert (scan_home / "output.txt").is_file(), "output.txt not found" assert (scan_home / "output.csv").is_file(), "output.csv not found" assert (scan_home / "output.ndjson").is_file(), "output.ndjson not found" + + with open(scan_home / "preset.yml") as f: + text = f.read() + assert " dns_resolution: false" in text + with open(scan_home / "output.csv") as f: lines = f.readlines() assert lines[0] == "Event type,Event data,IP Address,Source Module,Scope Distance,Event Tags\n" @@ -248,10 +254,13 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): result = await cli._main() assert result == True - # resolved dependency, excluded module + # enable and exclude the same module + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-m", "ffuf_shortnames", "-em", "ffuf_shortnames"]) result = await cli._main() - assert result == True + assert result == None + assert 'Unable to add scan module "ffuf_shortnames" because the module has been excluded' in caplog.text # require flags monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "-rf", "passive"]) @@ -269,9 +278,12 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): assert result == True # deadly modules + caplog.clear() + assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei"]) result = await cli._main() assert result == False, "-m nuclei ran without --allow-deadly" + assert "Please specify --allow-deadly to continue" in caplog.text # --allow-deadly monkeypatch.setattr("sys.argv", ["bbot", "-m", "nuclei", "--allow-deadly"]) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 39873ae0b..3b9f57767 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -304,7 +304,7 @@ def custom_setup(scan): # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0 events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.1/31", - modules=["httpx", "excavate"], + modules=["httpx"], _config={ "scope_search_distance": 0, "scope_dns_search_distance": 2, @@ -374,11 +374,12 @@ def custom_setup(scan): # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0, in_scope_only = False events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.1/31", - modules=["httpx", "excavate"], + modules=["httpx"], _config={ "scope_search_distance": 0, "scope_dns_search_distance": 2, "scope_report_distance": 1, + "excavate": True, "speculate": True, "modules": {"httpx": {"in_scope_only": False}, "speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], @@ -457,11 +458,12 @@ def custom_setup(scan): # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 1 events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.1/31", - modules=["httpx", "excavate"], + modules=["httpx"], _config={ "scope_search_distance": 1, "scope_dns_search_distance": 2, "scope_report_distance": 1, + "excavate": True, "speculate": True, "modules": {"httpx": {"in_scope_only": False}, "speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], @@ -551,12 +553,13 @@ def custom_setup(scan): events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.111/31", whitelist=["127.0.0.111/31", "127.0.0.222", "127.0.0.33"], - modules=["httpx", "excavate"], + modules=["httpx"], output_modules=["python"], _config={ "scope_search_distance": 0, "scope_dns_search_distance": 2, "scope_report_distance": 0, + "excavate": True, "speculate": True, "modules": {"speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], @@ -679,8 +682,8 @@ def custom_setup(scan): # sslcert with in-scope chain events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.0/31", - modules=["speculate", "sslcert"], - _config={"dns_resolution": False, "scope_report_distance": 0, "modules": {"speculate": {"ports": "9999"}}}, + modules=["sslcert"], + _config={"dns_resolution": False, "scope_report_distance": 0, "speculate": True, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) @@ -735,9 +738,9 @@ def custom_setup(scan): # sslcert with out-of-scope chain events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.0/31", - modules=["speculate", "sslcert"], + modules=["sslcert"], whitelist=["127.0.1.0"], - _config={"dns_resolution": False, "scope_report_distance": 0, "modules": {"speculate": {"ports": "9999"}}}, + _config={"dns_resolution": False, "scope_report_distance": 0, "speculate": True, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 8f696c1ac..2fada7968 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -80,8 +80,9 @@ async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock): base_output_module.watched_events = ["IP_ADDRESS"] scan2 = bbot_scanner( - modules=list(set(available_modules + available_internal_modules)), + modules=list(available_modules), output_modules=list(available_output_modules), + config={i: True for i in available_internal_modules}, force_start=True, ) scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers @@ -266,7 +267,8 @@ class mod_domain_only(BaseModule): async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_scanner, httpx_mock, monkeypatch): per_domain_scan = bbot_scanner( "evilcorp.com", - modules=list(set(available_modules + available_internal_modules)), + modules=list(available_modules), + config={i: True for i in available_internal_modules}, force_start=True, ) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 03c01b61d..864f1d0c4 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -88,6 +88,7 @@ def test_preset_yaml(clean_default_config): config={"preset_test_asdf": 1}, strict_scope=False, ) + preset1.bake() assert "evilcorp.com" in preset1.target assert "evilcorp.ce" in preset1.whitelist assert "test.www.evilcorp.ce" in preset1.blacklist @@ -257,112 +258,54 @@ def test_preset_module_resolution(clean_default_config): assert preset.modules == set(preset.output_modules).union(set(preset.internal_modules)) # make sure dependency resolution works as expected - preset.modules = ["wappalyzer"] + preset = Preset(modules=["wappalyzer"]).bake() assert set(preset.scan_modules) == {"wappalyzer", "httpx"} # make sure flags work as expected - preset = Preset() - assert not preset.flags - assert not preset.scan_modules - preset.flags = ["subdomain-enum"] + preset = Preset(flags=["subdomain-enum"]).bake() + assert preset.flags == {"subdomain-enum"} assert "sslcert" in preset.modules assert "wayback" in preset.modules assert "sslcert" in preset.scan_modules assert "wayback" in preset.scan_modules - # make sure module exclusions work as expected - preset.exclude_modules = ["sslcert"] - assert "sslcert" not in preset.modules - assert "wayback" in preset.modules - assert "sslcert" not in preset.scan_modules - assert "wayback" in preset.scan_modules - preset.scan_modules = ["sslcert"] - assert "sslcert" not in preset.modules - assert "wayback" not in preset.modules - assert "sslcert" not in preset.scan_modules - assert "wayback" not in preset.scan_modules - preset.exclude_modules = [] - preset.scan_modules = ["sslcert"] - assert "sslcert" in preset.modules - assert "wayback" not in preset.modules - assert "sslcert" in preset.scan_modules - assert "wayback" not in preset.scan_modules - preset.add_module("wayback") - assert "sslcert" in preset.modules - assert "wayback" in preset.modules - assert "sslcert" in preset.scan_modules - assert "wayback" in preset.scan_modules - preset.exclude_modules = ["sslcert"] + # flag + module exclusions + preset = Preset(flags=["subdomain-enum"], exclude_modules=["sslcert"]).bake() assert "sslcert" not in preset.modules assert "wayback" in preset.modules assert "sslcert" not in preset.scan_modules assert "wayback" in preset.scan_modules - # make sure flag requirements work as expected - preset = Preset() - preset.require_flags = ["passive"] - preset.scan_modules = ["sslcert"] - assert not preset.scan_modules - preset.scan_modules = ["wappalyzer"] - assert not preset.scan_modules - preset.flags = ["subdomain-enum"] - assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules + # flag + flag exclusions + preset = Preset(flags=["subdomain-enum"], exclude_flags=["active"]).bake() assert "sslcert" not in preset.modules - assert "sslcert" not in preset.scan_modules - preset.require_flags = [] assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules - assert "sslcert" not in preset.modules assert "sslcert" not in preset.scan_modules - assert not preset.require_flags - preset.flags = [] - preset.scan_modules = [] - assert not preset.flags - assert not preset.scan_modules - preset.scan_modules = ["sslcert", "wayback"] - assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules - assert "sslcert" in preset.modules - assert "sslcert" in preset.scan_modules - preset.require_flags = ["passive"] - assert "wayback" in preset.modules assert "wayback" in preset.scan_modules - assert "sslcert" not in preset.modules - assert "sslcert" not in preset.scan_modules - # make sure flag exclusions work as expected - preset = Preset() - preset.exclude_flags = ["active"] - preset.scan_modules = ["sslcert"] - assert not preset.scan_modules - preset.scan_modules = ["wappalyzer"] - assert not preset.scan_modules - preset.flags = ["subdomain-enum"] - assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules + # flag + flag requirements + preset = Preset(flags=["subdomain-enum"], require_flags=["passive"]).bake() assert "sslcert" not in preset.modules - assert "sslcert" not in preset.scan_modules - preset.exclude_flags = [] assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules - assert "sslcert" not in preset.modules assert "sslcert" not in preset.scan_modules - assert not preset.require_flags - preset.flags = [] - preset.scan_modules = [] - assert not preset.flags - assert not preset.scan_modules - preset.scan_modules = ["sslcert", "wayback"] - assert "wayback" in preset.modules - assert "wayback" in preset.scan_modules - assert "sslcert" in preset.modules - assert "sslcert" in preset.scan_modules - preset.exclude_flags = ["active"] - assert "wayback" in preset.modules assert "wayback" in preset.scan_modules - assert "sslcert" not in preset.modules - assert "sslcert" not in preset.scan_modules + + # normal module enableement + preset = Preset(modules=["sslcert", "wappalyzer", "wayback"]).bake() + assert set(preset.scan_modules) == {"sslcert", "wappalyzer", "wayback", "httpx"} + + # modules + flag exclusions + preset = Preset(exclude_flags=["active"], modules=["sslcert", "wappalyzer", "wayback"]).bake() + assert set(preset.scan_modules) == {"wayback"} + + # modules + flag requirements + preset = Preset(require_flags=["passive"], modules=["sslcert", "wappalyzer", "wayback"]).bake() + assert set(preset.scan_modules) == {"wayback"} + + # modules + module exclusions + with pytest.raises(ValidationError) as error: + preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake() + assert str(error.value) == 'Unable to add scan module "sslcert" because the module has been excluded' def test_preset_module_loader(): @@ -638,7 +581,7 @@ def get_module_flags(p): yield m, preloaded.get("flags", []) # enable by flag, no exclusions/requirements - preset = Preset(flags=["subdomain-enum"]) + preset = Preset(flags=["subdomain-enum"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) massdns_flags = preset.preloaded_module("massdns").get("flags", []) @@ -656,7 +599,7 @@ def get_module_flags(p): assert any("aggressive" in flags for module, flags in module_flags) # enable by flag, one required flag - preset = Preset(flags=["subdomain-enum"], require_flags=["passive"]) + preset = Preset(flags=["subdomain-enum"], require_flags=["passive"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert "massdns" in [x[0] for x in module_flags] @@ -666,7 +609,7 @@ def get_module_flags(p): assert any("aggressive" in flags for module, flags in module_flags) # enable by flag, one excluded flag - preset = Preset(flags=["subdomain-enum"], exclude_flags=["active"]) + preset = Preset(flags=["subdomain-enum"], exclude_flags=["active"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert "massdns" in [x[0] for x in module_flags] @@ -676,7 +619,7 @@ def get_module_flags(p): assert any("aggressive" in flags for module, flags in module_flags) # enable by flag, one excluded module - preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns"]) + preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert not "massdns" in [x[0] for x in module_flags] @@ -686,7 +629,7 @@ def get_module_flags(p): assert any("aggressive" in flags for module, flags in module_flags) # enable by flag, multiple required flags - preset = Preset(flags=["subdomain-enum"], require_flags=["safe", "passive"]) + preset = Preset(flags=["subdomain-enum"], require_flags=["safe", "passive"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert not "massdns" in [x[0] for x in module_flags] @@ -696,7 +639,7 @@ def get_module_flags(p): assert not any("aggressive" in flags for module, flags in module_flags) # enable by flag, multiple excluded flags - preset = Preset(flags=["subdomain-enum"], exclude_flags=["aggressive", "active"]) + preset = Preset(flags=["subdomain-enum"], exclude_flags=["aggressive", "active"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert not "massdns" in [x[0] for x in module_flags] @@ -706,7 +649,7 @@ def get_module_flags(p): assert not any("aggressive" in flags for module, flags in module_flags) # enable by flag, multiple excluded modules - preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns", "c99"]) + preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns", "c99"]).bake() assert len(preset.modules) > 25 module_flags = list(get_module_flags(preset)) assert not "massdns" in [x[0] for x in module_flags] diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 11e7a35ec..4d2b967c5 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -64,3 +64,50 @@ def test_python_api_sync(): bbot_home = "/tmp/.bbot_python_api_test" Scanner("127.0.0.1", config={"home": bbot_home}) assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools") + + +def test_python_api_validation(): + from bbot.scanner import Scanner, Preset + + # invalid module + with pytest.raises(ValidationError) as error: + Scanner(modules=["asdf"]) + assert str(error.value) == 'Could not find scan module "asdf". Did you mean "asn"?' + # invalid output module + with pytest.raises(ValidationError) as error: + Scanner(output_modules=["asdf"]) + assert str(error.value) == 'Could not find output module "asdf". Did you mean "teams"?' + # invalid excluded module + with pytest.raises(ValidationError) as error: + Scanner(exclude_modules=["asdf"]) + assert str(error.value) == 'Could not find module "asdf". Did you mean "asn"?' + # invalid flag + with pytest.raises(ValidationError) as error: + Scanner(flags=["asdf"]) + assert str(error.value) == 'Could not find flag "asdf". Did you mean "safe"?' + # invalid required flag + with pytest.raises(ValidationError) as error: + Scanner(require_flags=["asdf"]) + assert str(error.value) == 'Could not find flag "asdf". Did you mean "safe"?' + # invalid excluded flag + with pytest.raises(ValidationError) as error: + Scanner(exclude_flags=["asdf"]) + assert str(error.value) == 'Could not find flag "asdf". Did you mean "safe"?' + # output module as normal module + with pytest.raises(ValidationError) as error: + Scanner(modules=["json"]) + assert str(error.value) == 'Could not find scan module "json". Did you mean "asn"?' + # normal module as output module + with pytest.raises(ValidationError) as error: + Scanner(output_modules=["robots"]) + assert str(error.value) == 'Could not find output module "robots". Did you mean "web_report"?' + # invalid preset type + with pytest.raises(ValidationError) as error: + Scanner(preset="asdf") + assert str(error.value) == 'Preset must be of type Preset, not "str"' + # include nonexistent preset + with pytest.raises(ValidationError) as error: + Preset(include=["asdf"]) + assert ( + str(error.value) == 'Could not find preset at "asdf" - file does not exist. Use -lp to list available presets' + ) From 86b798e7da70b8c95b4873d9795623cf91156dde Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 29 Mar 2024 17:14:12 -0400 Subject: [PATCH 079/171] blacked --- bbot/core/core.py | 1 + bbot/scanner/preset/preset.py | 68 +++++++++++++++++++++++++++++++++- docs/index.md | 2 +- docs/scanning/configuration.md | 18 ++++----- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index e8170c4eb..9cc7d4c88 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -7,6 +7,7 @@ class BBOTCore: + """ """ def __init__(self): self._logger = None diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index d8c068f29..138d50831 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -132,7 +132,7 @@ def __init__( verbose (bool, optional): Set the BBOT logger to verbose mode. debug (bool, optional): Set the BBOT logger to debug mode. silent (bool, optional): Silence all stderr (effectively disables the BBOT logger). - _exclude (bool, optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets. + _exclude (list[Path], optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets. _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc. """ # internal variables @@ -331,7 +331,7 @@ def bake(self): """ Return a "baked" copy of this preset, ready for use by a BBOT scan. - Baking a preset finalizes it by populating `preset.modules` based on flags, + Baking a preset finalizes it by populating `preset.modules` based on flags, performing final validations, and substituting environment variables in preloaded modules. It also evaluates custom `conditions` as specified in the preset. @@ -569,6 +569,21 @@ def whitelisted(self, e): @classmethod def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False): + """ + Create a preset from a Python dictionary object. + + Args: + preset_dict (dict): Preset in dictionary form + name (str, optional): Name of preset + _exclude (list[Path], optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets. + _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc. + + Returns: + Preset: The loaded preset + + Examples: + >>> preset = Preset.from_dict({"target": "evilcorp.com", "modules": ["nmap}]}) + """ new_preset = cls( *preset_dict.get("target", []), whitelist=preset_dict.get("whitelist"), @@ -597,6 +612,12 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False): return new_preset def include_preset(self, filename): + """ + Load a preset from a yaml file and merge it into this one + + Args: + filename (Path): The preset YAML file to merge + """ self.log_debug(f'Including preset "{filename}"') preset_filename = PRESET_PATH.find(filename) preset_from_yaml = self.from_yaml_file(preset_filename, _exclude=self._preset_files_loaded) @@ -633,6 +654,21 @@ def from_yaml_string(cls, yaml_preset): return cls.from_dict(omegaconf.OmegaConf.create(yaml_preset)) def to_dict(self, include_target=False, full_config=False): + """ + Convert this preset into a Python dictionary. + + Args: + include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary + full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults. + + Returns: + dict: The preset in dictionary form + + Example: + >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"]) + >>> preset.to_dict() + {"flags": ["subdomain-enum"], "modules": ["nmap"]} + """ preset_dict = {} # config @@ -693,6 +729,25 @@ def to_dict(self, include_target=False, full_config=False): return preset_dict def to_yaml(self, include_target=False, full_config=False, sort_keys=False): + """ + Return the preset in the form of a YAML string. + + Args: + include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary + full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults. + sort_keys (bool, optional): If True, sort YAML keys alphabetically + + Returns: + str: The preset in the form of a YAML string + + Example: + >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"]) + >>> print(preset.to_yaml()) + flags: + - subdomain-enum + modules: + - nmap + """ preset_dict = self.to_dict(include_target=include_target, full_config=full_config) return yaml.dump(preset_dict, sort_keys=sort_keys) @@ -743,6 +798,9 @@ def _is_valid_module(self, module, module_type, name_only=False, raise_error=Tru return True, "", preloaded def validate(self): + """ + Validate module/flag exclusions/requirements, and CLI config options if applicable. + """ if self._cli: self.args.validate() @@ -767,6 +825,9 @@ def validate(self): @property def all_presets(self): + """ + Recursively find all the presets and return them as a dictionary + """ preset_dir = self.preset_dir home_dir = Path.home() @@ -822,6 +883,9 @@ def all_presets(self): return DEFAULT_PRESETS def presets_table(self, include_modules=True): + """ + Return a table of all the presets in the form of a string + """ table = [] header = ["Preset", "Category", "Description", "# Modules"] if include_modules: diff --git a/docs/index.md b/docs/index.md index 7d9894126..7e7adb510 100644 --- a/docs/index.md +++ b/docs/index.md @@ -117,7 +117,7 @@ config: Note: this will ensure the API keys are used in all scans, regardless of preset. -```yaml title="~/.config/bbot/secrets.yml" +```yaml title="~/.config/bbot/bbot.yml" modules: shodan_dns: api_key: deadbeef diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index c27a42f73..0cc7dddab 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -1,6 +1,6 @@ # Configuration Overview -Normally, [Presets](presets.md) are used to configure a scan. However, there may be cases where you want to change BBOT's defaults so a certain option is always set, even if it's not specified in a preset. +Normally, [Presets](presets.md) are used to configure a scan. However, there may be cases where you want to change BBOT's global defaults so a certain option is always set, even if it's not specified in a preset. BBOT has a YAML config at `~/.config/bbot.yml`. This is the first config that BBOT loads, so it's a good place to put default settings like `http_proxy`, `max_threads`, or `http_user_agent`. You can also put any module settings here, including **API keys**. @@ -13,13 +13,13 @@ For examples of common config changes, see [Tips and Tricks](tips_and_tricks.md) ## Configuration Files -BBOT loads its config from the following files, in this order: +BBOT loads its config from the following files, in this order (last one loaded == highest priority): -- `~/.config/bbot/bbot.yml` <-- Use this one as your main config -- `~/.config/bbot/secrets.yml` <-- Use this one for sensitive stuff like API keys -- command line (`--config`) <-- Use this to specify a custom config file or override individual config options +- `~/.config/bbot/bbot.yml` <-- Global BBOT config +- presets (`-p`) <-- Presets are good for scan-specific settings +- command line (`-c`) <-- CLI overrides everything -These config files will be automatically created for you when you first run BBOT. +`bbot.yml` will be automatically created for you when you first run BBOT. ## YAML Config vs Command Line @@ -27,7 +27,7 @@ You can specify config options either via the command line or the config. For ex ```bash # send BBOT traffic through an HTTP proxy -bbot -t evilcorp.com --config http_proxy=http://127.0.0.1:8080 +bbot -t evilcorp.com -c http_proxy=http://127.0.0.1:8080 ``` Or, in `~/.config/bbot/config.yml`: @@ -38,7 +38,7 @@ http_proxy: http://127.0.0.1:8080 These two are equivalent. -Config options specified via the command-line take precedence over all others. You can give BBOT a custom config file with `--config myconf.yml`, or individual arguments like this: `--config modules.shodan_dns.api_key=deadbeef`. To display the full and current BBOT config, including any command-line arguments, use `bbot --current-config`. +Config options specified via the command-line take precedence over all others. You can give BBOT a custom config file with `-c myconf.yml`, or individual arguments like this: `-c modules.shodan_dns.api_key=deadbeef`. To display the full and current BBOT config, including any command-line arguments, use `bbot -c`. Note that placing the following in `bbot.yml`: ```yaml title="~/.bbot/config/bbot.yml" @@ -48,7 +48,7 @@ modules: ``` Is the same as: ```bash -bbot --config modules.shodan_dns.api_key=deadbeef +bbot -c modules.shodan_dns.api_key=deadbeef ``` ## Global Config Options From b7cacdb3f2ad8e9a7563c87c14cb136015aac70c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 29 Mar 2024 17:38:24 -0400 Subject: [PATCH 080/171] more docs --- bbot/core/config/logger.py | 6 ++++++ bbot/core/core.py | 31 ++++++++++++++++++++++++++++++- bbot/scanner/preset/preset.py | 2 +- mkdocs.yml | 6 ++++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 31ca4184e..a4063027b 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -34,6 +34,12 @@ def format(self, record): class BBOTLogger: + """ + The main BBOT logger. + + The job of this class is to manage the different log handlers in BBOT, + allow adding new log handlers, and easily switching log levels on the fly. + """ def __init__(self, core): # custom logging levels diff --git a/bbot/core/core.py b/bbot/core/core.py index 9cc7d4c88..313aa236a 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -7,7 +7,18 @@ class BBOTCore: - """ """ + """ + This is the first thing that loads when you import BBOT. + + Unlike a Preset, BBOTCore holds only the config, not scan-specific stuff like targets, flags, modules, etc. + + Its main jobs are: + + - set up logging + - keep separation between the `default` and `custom` config (this allows presets to only display the config options that have changed) + - allow for easy merging of configs + - load quickly + """ def __init__(self): self._logger = None @@ -65,6 +76,9 @@ def config(self): @property def default_config(self): + """ + The default BBOT config (from `defaults.yml`). Read-only. + """ global DEFAULT_CONFIG if DEFAULT_CONFIG is None: self.default_config = self.files_config.get_default_config() @@ -81,6 +95,9 @@ def default_config(self, value): @property def custom_config(self): + """ + Custom BBOT config (from `~/.config/bbot/bbot.yml`) + """ # we temporarily clear out the config so it can be refreshed if/when custom_config changes self._config = None if self._custom_config is None: @@ -94,18 +111,30 @@ def custom_config(self, value): self._custom_config = value def merge_custom(self, config): + """ + Merge a config into the custom config. + """ self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) def merge_default(self, config): + """ + Merge a config into the default config. + """ self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config)) def copy(self): + """ + Return a semi-shallow copy of self. (`custom_config` is copied, but `default_config` stays the same) + """ core_copy = copy(self) core_copy._custom_config = self._custom_config.copy() return core_copy @property def files_config(self): + """ + Get the configs from `bbot.yml` and `defaults.yml` + """ if self._files_config is None: from .config import files diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 138d50831..ffb5ed868 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -362,7 +362,7 @@ def bake(self): for module in baked_preset.explicit_scan_modules: baked_preset.add_module(module, module_type="scan") # enable output modules - for module in self.explicit_output_modules: + for module in baked_preset.explicit_output_modules: baked_preset.add_module(module, module_type="output", raise_error=False) # enable internal modules diff --git a/mkdocs.yml b/mkdocs.yml index c2fea7968..6cb486614 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -33,18 +33,20 @@ nav: - List of Modules: modules/list_of_modules.md - Nuclei: modules/nuclei.md - Misc: + - Contribution: contribution.md - Release History: release_history.md - Troubleshooting: troubleshooting.md - Developer Manual: - Development Overview: dev/index.md - - How to Write a BBOT Module: contribution.md + - How to Write a BBOT Module: dev/module_howto.md - Discord Bot Example: dev/discord_bot.md - Code Reference: - - Presets: dev/presets.md - Scanner: dev/scanner.md + - Presets: dev/presets.md - Event: dev/event.md - Target: dev/target.md - BaseModule: dev/basemodule.md + - BBOTCore: dev/core.md - Helpers: - Overview: dev/helpers/index.md - Command: dev/helpers/command.md From 8ccd43972bd7c5fe1b9692d7b05fc0e8a92ab232 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 29 Mar 2024 17:38:39 -0400 Subject: [PATCH 081/171] add core.md --- docs/dev/core.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dev/core.md diff --git a/docs/dev/core.md b/docs/dev/core.md new file mode 100644 index 000000000..d138681f9 --- /dev/null +++ b/docs/dev/core.md @@ -0,0 +1 @@ +::: bbot.core.core.BBOTCore From fd258ba2641db41f08a649488620dbdca467686d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 29 Mar 2024 17:52:32 -0400 Subject: [PATCH 082/171] fix docs --- bbot/scanner/preset/preset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index ffb5ed868..9431f53be 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -277,7 +277,7 @@ def merge(self, other): Args: other (Preset): The preset to merge into this one. - Example: + Examples: >>> preset1 = Preset(modules=["nmap"]) >>> preset1.scan_modules ['nmap'] @@ -664,7 +664,7 @@ def to_dict(self, include_target=False, full_config=False): Returns: dict: The preset in dictionary form - Example: + Examples: >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"]) >>> preset.to_dict() {"flags": ["subdomain-enum"], "modules": ["nmap"]} @@ -740,7 +740,7 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): Returns: str: The preset in the form of a YAML string - Example: + Examples: >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"]) >>> print(preset.to_yaml()) flags: From 781bcc4769cfc7017d81acb4d87adaaa497f05fe Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 30 Mar 2024 23:10:08 -0400 Subject: [PATCH 083/171] more docstrings --- bbot/scanner/preset/preset.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9431f53be..7b057aa02 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -533,13 +533,13 @@ def args(self): self._args = BBOTArgs(self) return self._args - def in_scope(self, e): + def in_scope(self, host): """ Check whether a hostname, url, IP, etc. is in scope. Accepts either events or string data. Checks whitelist and blacklist. - If `e` is an event and its scope distance is zero, it will be considered in-scope. + If `host` is an event and its scope distance is zero, it will automatically be considered in-scope. Examples: Check if a URL is in scope: @@ -553,16 +553,36 @@ def in_scope(self, e): in_scope = e.scope_distance == 0 or self.whitelisted(e) return in_scope and not self.blacklisted(e) - def blacklisted(self, e): + def blacklisted(self, host): """ Check whether a hostname, url, IP, etc. is blacklisted. + + Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute. + + Args: + host (str or IPAddress or Event): The host to check against the blacklist + + Examples: + Check if a URL's host is blacklisted: + >>> preset.blacklisted("http://www.evilcorp.com") + True """ e = make_event(e, dummy=True) return e in self.blacklist - def whitelisted(self, e): + def whitelisted(self, host): """ Check whether a hostname, url, IP, etc. is whitelisted. + + Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute. + + Args: + host (str or IPAddress or Event): The host to check against the whitelist + + Examples: + Check if a URL's host is whitelisted: + >>> preset.whitelisted("http://www.evilcorp.com") + True """ e = make_event(e, dummy=True) return e in self.whitelist From 2a735affa261e1d9c97ff727164db85cdf6db9be Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 31 Mar 2024 13:07:31 -0400 Subject: [PATCH 084/171] rename output.ndjson --> output.json --- bbot/modules/output/json.py | 2 +- bbot/test/test_step_1/test_cli.py | 2 +- bbot/test/test_step_1/test_python_api.py | 6 +++--- bbot/test/test_step_2/module_tests/test_module_json.py | 4 ++-- docs/scanning/output.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index bf8517db9..0b7a16a5f 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -16,7 +16,7 @@ class JSON(BaseOutputModule): _preserve_graph = True async def setup(self): - self._prep_output_dir("output.ndjson") + self._prep_output_dir("output.json") self.siem_friendly = self.config.get("siem_friendly", False) return True diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index dce77371c..6ada6e64e 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -24,7 +24,7 @@ async def test_cli_scan(monkeypatch): assert (scan_home / "wordcloud.tsv").is_file(), "wordcloud.tsv not found" assert (scan_home / "output.txt").is_file(), "output.txt not found" assert (scan_home / "output.csv").is_file(), "output.csv not found" - assert (scan_home / "output.ndjson").is_file(), "output.ndjson not found" + assert (scan_home / "output.json").is_file(), "output.json not found" with open(scan_home / "preset.yml") as f: text = f.read() diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 4d2b967c5..14c2fae1d 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -15,7 +15,7 @@ async def test_python_api(): scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") await scan2.async_start_without_generator() scan_home = scan2.helpers.scans_dir / "python_api_test" - out_file = scan_home / "output.ndjson" + out_file = scan_home / "output.json" assert list(scan2.helpers.read_file(out_file)) scan_log = scan_home / "scan.log" debug_log = scan_home / "debug.log" @@ -31,7 +31,7 @@ async def test_python_api(): assert "scan_logging_test" not in open(debug_log).read() scan_home = scan3.helpers.scans_dir / "scan_logging_test" - out_file = scan_home / "output.ndjson" + out_file = scan_home / "output.json" assert list(scan3.helpers.read_file(out_file)) scan_log = scan_home / "scan.log" debug_log = scan_home / "debug.log" @@ -58,7 +58,7 @@ def test_python_api_sync(): # make sure output files work scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") scan2.start_without_generator() - out_file = scan2.helpers.scans_dir / "python_api_test" / "output.ndjson" + out_file = scan2.helpers.scans_dir / "python_api_test" / "output.json" assert list(scan2.helpers.read_file(out_file)) # make sure config loads properly bbot_home = "/tmp/.bbot_python_api_test" diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 03b1f6a24..53affcd30 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -6,7 +6,7 @@ class TestJSON(ModuleTestBase): def check(self, module_test, events): - txt_file = module_test.scan.home / "output.ndjson" + txt_file = module_test.scan.home / "output.json" lines = list(module_test.scan.helpers.read_file(txt_file)) assert lines e = event_from_json(json.loads(lines[0])) @@ -19,7 +19,7 @@ class TestJSONSIEMFriendly(ModuleTestBase): config_overrides = {"modules": {"json": {"siem_friendly": True}}} def check(self, module_test, events): - txt_file = module_test.scan.home / "output.ndjson" + txt_file = module_test.scan.home / "output.json" lines = list(module_test.scan.helpers.read_file(txt_file)) passed = False for line in lines: diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 4acd25250..d51bc4865 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -64,7 +64,7 @@ You can filter on the JSON output with `jq`: ```bash # pull out only the .data attribute of every DNS_NAME -$ jq -r 'select(.type=="DNS_NAME") | .data' ~/.bbot/scans/extreme_johnny/output.ndjson +$ jq -r 'select(.type=="DNS_NAME") | .data' ~/.bbot/scans/extreme_johnny/output.json evilcorp.com www.evilcorp.com mail.evilcorp.com From da477e10bff1945adcc8be09be69db179c4be700 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 31 Mar 2024 13:09:25 -0400 Subject: [PATCH 085/171] fix tests --- bbot/scanner/preset/preset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 7b057aa02..09de8bed7 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -547,7 +547,7 @@ def in_scope(self, host): True """ try: - e = make_event(e, dummy=True) + e = make_event(host, dummy=True) except ValidationError: return False in_scope = e.scope_distance == 0 or self.whitelisted(e) @@ -567,7 +567,7 @@ def blacklisted(self, host): >>> preset.blacklisted("http://www.evilcorp.com") True """ - e = make_event(e, dummy=True) + e = make_event(host, dummy=True) return e in self.blacklist def whitelisted(self, host): @@ -584,7 +584,7 @@ def whitelisted(self, host): >>> preset.whitelisted("http://www.evilcorp.com") True """ - e = make_event(e, dummy=True) + e = make_event(host, dummy=True) return e in self.whitelist @classmethod From 24acdb7a8304c1936321454f95fdf2ee71b71a77 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 12:14:52 -0400 Subject: [PATCH 086/171] let there be dns engine --- bbot/core/config/__init__.py | 12 + bbot/core/config/logger.py | 36 +- bbot/core/core.py | 27 ++ bbot/core/engine.py | 189 ++++++++++ bbot/core/helpers/dns/__init__.py | 1 + bbot/core/helpers/dns/dns.py | 234 +++++++++++++ bbot/core/helpers/{dns.py => dns/engine.py} | 326 +++++++----------- bbot/core/helpers/dns/mock.py | 55 +++ bbot/core/helpers/misc.py | 27 ++ bbot/scanner/scanner.py | 8 +- bbot/test/bbot_fixtures.py | 70 +--- bbot/test/test_step_1/test_dns.py | 146 +++++--- .../test_step_1/test_manager_deduplication.py | 2 +- .../test_manager_scope_accuracy.py | 2 +- bbot/test/test_step_1/test_modules_basic.py | 3 +- bbot/test/test_step_1/test_scan.py | 4 +- bbot/test/test_step_2/module_tests/base.py | 4 +- .../module_tests/test_module_affiliates.py | 2 +- .../module_tests/test_module_aggregate.py | 2 +- .../test_module_asset_inventory.py | 2 +- .../module_tests/test_module_baddns.py | 4 +- .../module_tests/test_module_baddns_zone.py | 4 +- .../module_tests/test_module_dehashed.py | 2 +- .../module_tests/test_module_dnscommonsrv.py | 2 +- .../module_tests/test_module_internetdb.py | 2 +- .../module_tests/test_module_ipneighbor.py | 2 +- .../module_tests/test_module_postman.py | 2 +- .../module_tests/test_module_speculate.py | 2 +- 28 files changed, 823 insertions(+), 349 deletions(-) create mode 100644 bbot/core/engine.py create mode 100644 bbot/core/helpers/dns/__init__.py create mode 100644 bbot/core/helpers/dns/dns.py rename bbot/core/helpers/{dns.py => dns/engine.py} (78%) create mode 100644 bbot/core/helpers/dns/mock.py diff --git a/bbot/core/config/__init__.py b/bbot/core/config/__init__.py index e69de29bb..c36d91f48 100644 --- a/bbot/core/config/__init__.py +++ b/bbot/core/config/__init__.py @@ -0,0 +1,12 @@ +import sys +import multiprocessing as mp + +try: + mp.set_start_method("spawn") +except Exception: + start_method = mp.get_start_method() + if start_method != "spawn": + print( + f"[WARN] Multiprocessing spawn method is set to {start_method}. This may negatively affect performance.", + file=sys.stderr, + ) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index a4063027b..3844a65fc 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -1,6 +1,8 @@ import sys +import atexit import logging from copy import copy +import multiprocessing import logging.handlers from pathlib import Path @@ -60,13 +62,29 @@ def __init__(self, core): self.core_logger = logging.getLogger("bbot") self.core = core - # Don't do this more than once - if len(self.core_logger.handlers) == 0: - for logger in self.loggers: - self.include_logger(logger) + self.process_name = multiprocessing.current_process().name + if self.process_name == "MainProcess": + self.queue = multiprocessing.Queue() + self.setup_queue_handler() + # Start the QueueListener + self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values()) + self.listener.start() + atexit.register(self.listener.stop) self.log_level = logging.INFO + def setup_queue_handler(self, logging_queue=None): + if logging_queue is None: + logging_queue = self.queue + else: + self.queue = logging_queue + self.queue_handler = logging.handlers.QueueHandler(logging_queue) + logging.getLogger().addHandler(self.queue_handler) + self.core_logger.setLevel(self.log_level) + # disable asyncio logging for child processes + if self.process_name != "MainProcess": + logging.getLogger("asyncio").setLevel(logging.ERROR) + def addLoggingLevel(self, levelName, levelNum, methodName=None): """ Comprehensively adds a new logging level to the `logging` module and the @@ -127,15 +145,19 @@ def loggers(self): return self._loggers def add_log_handler(self, handler, formatter=None): + if self.listener is None: + return if handler.formatter is None: handler.setFormatter(debug_format) for logger in self.loggers: - if handler not in logger.handlers: + if handler not in self.listener.handlers: logger.addHandler(handler) def remove_log_handler(self, handler): + if self.listener is None: + return for logger in self.loggers: - if handler in logger.handlers: + if handler in self.listener.handlers: logger.removeHandler(handler) def include_logger(self, logger): @@ -144,7 +166,7 @@ def include_logger(self, logger): if self.log_level is not None: logger.setLevel(self.log_level) for handler in self.log_handlers.values(): - logger.addHandler(handler) + self.add_log_handler(handler) @property def log_handlers(self): diff --git a/bbot/core/core.py b/bbot/core/core.py index 313aa236a..a38307763 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,5 +1,7 @@ import logging +import traceback from copy import copy +import multiprocessing from pathlib import Path from omegaconf import OmegaConf @@ -20,6 +22,26 @@ class BBOTCore: - load quickly """ + class BBOTProcess(multiprocessing.Process): + + def __init__(self, *args, **kwargs): + self.logging_queue = kwargs.pop("logging_queue") + super().__init__(*args, **kwargs) + + def run(self): + log = logging.getLogger("bbot.core.process") + try: + from bbot.core import CORE + + CORE.logger.setup_queue_handler(self.logging_queue) + super().run() + except KeyboardInterrupt: + log.warning(f"Got KeyboardInterrupt in {self.name}") + log.trace(traceback.format_exc()) + except BaseException as e: + log.warning(f"Error in {self.name}: {e}") + log.trace(traceback.format_exc()) + def __init__(self): self._logger = None self._files_config = None @@ -142,6 +164,11 @@ def files_config(self): self._files_config = files.BBOTConfigFiles(self) return self._files_config + def create_process(self, *args, **kwargs): + process = self.BBOTProcess(*args, logging_queue=self.logger.queue, **kwargs) + process.daemon = True + return process + @property def logger(self): self.config diff --git a/bbot/core/engine.py b/bbot/core/engine.py new file mode 100644 index 000000000..5f4530ba1 --- /dev/null +++ b/bbot/core/engine.py @@ -0,0 +1,189 @@ +import zmq +import atexit +import pickle +import asyncio +import inspect +import logging +import tempfile +import traceback +import zmq.asyncio +from pathlib import Path +from contextlib import contextmanager + +from bbot.core import CORE +from bbot.core.helpers.misc import rand_string + +CMD_EXIT = 1000 + + +class EngineClient: + + SERVER_CLASS = None + + def __init__(self, **kwargs): + self.name = f"EngineClient {self.__class__.__name__}" + if self.SERVER_CLASS is None: + raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}") + self.CMDS = dict(self.SERVER_CLASS.CMDS) + for k, v in list(self.CMDS.items()): + self.CMDS[v] = k + self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}") + self.socket_address = f"zmq_{rand_string(8)}.sock" + self.socket_path = Path(tempfile.gettempdir()) / self.socket_address + self.server_kwargs = kwargs.pop("server_kwargs", {}) + self.server_process = self.start_server() + self.context = zmq.asyncio.Context() + + async def run_and_return(self, command, **kwargs): + with self.new_socket() as socket: + message = self.make_message(command, args=kwargs) + await socket.send(message) + binary = await socket.recv() + self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") + message = pickle.loads(binary) + self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}") + # error handling + if self.check_error(message): + return + return message + + async def run_and_yield(self, command, **kwargs): + message = self.make_message(command, args=kwargs) + with self.new_socket() as socket: + await socket.send(message) + while 1: + binary = await socket.recv() + self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") + message = pickle.loads(binary) + self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}") + # error handling + if self.check_error(message) or self.check_stop(message): + break + yield message + + def check_error(self, message): + if isinstance(message, dict) and len(message) == 1 and "_e" in message: + error, trace = message["_e"] + self.log.error(error) + self.log.trace(trace) + return True + return False + + def check_stop(self, message): + if isinstance(message, dict) and len(message) == 1 and "_s" in message: + return True + return False + + def make_message(self, command, args): + try: + cmd_id = self.CMDS[command] + except KeyError: + raise KeyError(f'Command "{command}" not found. Available commands: {",".join(self.available_commands)}') + return pickle.dumps(dict(c=cmd_id, a=args)) + + @property + def available_commands(self): + return [s for s in self.CMDS if isinstance(s, str)] + + def start_server(self, **server_kwargs): + process = CORE.create_process( + target=self.server_process, + args=( + self.SERVER_CLASS, + self.socket_path, + ), + kwargs=self.server_kwargs, + ) + process.start() + return process + + @staticmethod + def server_process(server_class, socket_path, **kwargs): + engine_server = server_class(socket_path, **kwargs) + asyncio.run(engine_server.worker()) + + @contextmanager + def new_socket(self): + socket = self.context.socket(zmq.DEALER) + socket.connect(f"ipc://{self.socket_path}") + try: + yield socket + finally: + socket.close() + + +class EngineServer: + + CMDS = {} + + def __init__(self, socket_path): + self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}") + self.name = f"EngineServer {self.__class__.__name__}" + if socket_path is not None: + # create ZeroMQ context + self.context = zmq.asyncio.Context() + # ROUTER socket can handle multiple concurrent requests + self.socket = self.context.socket(zmq.ROUTER) + # create socket file + self.socket.bind(f"ipc://{socket_path}") + # delete socket file on exit + atexit.register(socket_path.unlink, missing_ok=True) + + async def run_and_return(self, client_id, command_fn, **kwargs): + self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({kwargs})") + try: + result = await command_fn(**kwargs) + except Exception as e: + error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}" + trace = traceback.format_exc() + result = {"_e": (error, trace)} + await self.socket.send_multipart([client_id, pickle.dumps(result)]) + + async def run_and_yield(self, client_id, command_fn, **kwargs): + self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({kwargs})") + try: + async for _ in command_fn(**kwargs): + await self.socket.send_multipart([client_id, pickle.dumps(_)]) + await self.socket.send_multipart([client_id, pickle.dumps({"_s": None})]) + except Exception as e: + error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}" + trace = traceback.format_exc() + result = {"_e": (error, trace)} + await self.socket.send_multipart([client_id, pickle.dumps(result)]) + + async def worker(self): + try: + while 1: + client_id, binary = await self.socket.recv_multipart() + self.log.debug(f"{self.name} got binary: {binary}") + message = pickle.loads(binary) + self.log.debug(f"{self.name} got message: {message}") + + cmd = message.get("c", None) + if not isinstance(cmd, int): + self.log.warning(f"No command sent in message: {message}") + continue + + kwargs = message.get("a", {}) + if not isinstance(kwargs, dict): + self.log.warning(f"{self.name}: received invalid message of type {type(kwargs)}, should be dict") + continue + + command_name = self.CMDS[cmd] + command_fn = getattr(self, command_name, None) + + if command_fn is None: + self.log.warning(f'{self.name} has no function named "{command_fn}"') + continue + + if inspect.isasyncgenfunction(command_fn): + coroutine = self.run_and_yield(client_id, command_fn, **kwargs) + else: + coroutine = self.run_and_return(client_id, command_fn, **kwargs) + + asyncio.create_task(coroutine) + except Exception as e: + self.log.error(f"Error in EngineServer worker: {e}") + self.log.trace(traceback.format_exc()) + finally: + self.socket.close() diff --git a/bbot/core/helpers/dns/__init__.py b/bbot/core/helpers/dns/__init__.py new file mode 100644 index 000000000..75426cd26 --- /dev/null +++ b/bbot/core/helpers/dns/__init__.py @@ -0,0 +1 @@ +from .dns import DNSHelper diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py new file mode 100644 index 000000000..1a9474277 --- /dev/null +++ b/bbot/core/helpers/dns/dns.py @@ -0,0 +1,234 @@ +import dns +import logging +import dns.exception +import dns.asyncresolver + +from bbot.core.engine import EngineClient +from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host + +from .engine import DNSEngine + +log = logging.getLogger("bbot.core.helpers.dns") + + +class DNSHelper(EngineClient): + + SERVER_CLASS = DNSEngine + + """Helper class for DNS-related operations within BBOT. + + This class provides mechanisms for host resolution, wildcard domain detection, event tagging, and more. + It centralizes all DNS-related activities in BBOT, offering both synchronous and asynchronous methods + for DNS resolution, as well as various utilities for batch resolution and DNS query filtering. + + Attributes: + parent_helper: A reference to the instantiated `ConfigAwareHelper` (typically `scan.helpers`). + resolver (BBOTAsyncResolver): An asynchronous DNS resolver tailored for BBOT with rate-limiting capabilities. + timeout (int): The timeout value for DNS queries. Defaults to 5 seconds. + retries (int): The number of retries for failed DNS queries. Defaults to 1. + abort_threshold (int): The threshold for aborting after consecutive failed queries. Defaults to 50. + max_dns_resolve_distance (int): Maximum allowed distance for DNS resolution. Defaults to 4. + all_rdtypes (list): A list of DNS record types to be considered during operations. + wildcard_ignore (tuple): Domains to be ignored during wildcard detection. + wildcard_tests (int): Number of tests to be run for wildcard detection. Defaults to 5. + _wildcard_cache (dict): Cache for wildcard detection results. + _dns_cache (LRUCache): Cache for DNS resolution results, limited in size. + _event_cache (LRUCache): Cache for event resolution results, tags. Limited in size. + resolver_file (Path): File containing system's current resolver nameservers. + filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True. + + Args: + parent_helper: The parent helper object with configuration details and utilities. + + Raises: + DNSError: If an issue arises when creating the BBOTAsyncResolver instance. + + Examples: + >>> dns_helper = DNSHelper(parent_config) + >>> resolved_host = dns_helper.resolver.resolve("example.com") + """ + + def __init__(self, parent_helper): + self.parent_helper = parent_helper + self.config = self.parent_helper.config + super().__init__(server_kwargs={"config": self.config}) + + # resolver + self.timeout = self.config.get("dns_timeout", 5) + self.resolver = dns.asyncresolver.Resolver() + self.resolver.rotate = True + self.resolver.timeout = self.timeout + self.resolver.lifetime = self.timeout + + self.max_dns_resolve_distance = self.config.get("max_dns_resolve_distance", 5) + + # wildcard handling + self.wildcard_ignore = self.config.get("dns_wildcard_ignore", None) + if not self.wildcard_ignore: + self.wildcard_ignore = [] + self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) + + # copy the system's current resolvers to a text file for tool use + self.system_resolvers = dns.resolver.Resolver().nameservers + # TODO: DNS server speed test (start in background task) + self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False) + + async def resolve(self, query, **kwargs): + return await self.run_and_return("resolve", query=query, **kwargs) + + async def resolve_batch(self, queries, **kwargs): + async for _ in self.run_and_yield("resolve_batch", queries=queries, **kwargs): + yield _ + + async def resolve_custom_batch(self, queries): + async for _ in self.run_and_yield("resolve_custom_batch", queries=queries): + yield _ + + async def resolve_event(self, event, minimal=False): + # abort if the event doesn't have a host + if (not event.host) or (event.type in ("IP_RANGE",)): + # tags, whitelisted, blacklisted, children + return set(), False, False, dict() + + event_host = str(event.host) + event_type = str(event.type) + kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} + return await self.run_and_return("resolve_event", **kwargs) + + async def is_wildcard(self, query, ips=None, rdtype=None): + """ + Use this method to check whether a *host* is a wildcard entry + + This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain. + + If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead. + + Args: + query (str): The hostname to check for a wildcard entry. + ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query. + rdtype (str, optional): The DNS record type (e.g., "A", "AAAA") to consider during the check. + + Returns: + dict: A dictionary indicating if the query is a wildcard for each checked DNS record type. + Keys are DNS record types like "A", "AAAA", etc. + Values are tuples where the first element is a boolean indicating if the query is a wildcard, + and the second element is the wildcard parent if it's a wildcard. + + Raises: + ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified. + + Examples: + >>> is_wildcard("www.github.io") + {"A": (True, "github.io"), "AAAA": (True, "github.io")} + + >>> is_wildcard("www.evilcorp.com", ips=["93.184.216.34"], rdtype="A") + {"A": (False, "evilcorp.com")} + + Note: + `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive) + """ + if [ips, rdtype].count(None) == 1: + raise ValueError("Both ips and rdtype must be specified") + + # skip if query isn't a dns name + if not is_dns_name(query): + return {} + + # skip check if the query's parent domain is excluded in the config + for d in self.wildcard_ignore: + if host_in_host(query, d): + log.debug(f"Skipping wildcard detection on {query} because it is excluded in the config") + return {} + + query = clean_dns_record(query) + # skip check if it's an IP or a plain hostname + if is_ip(query) or not "." in query: + return {} + # skip check if the query is a domain + if is_domain(query): + return {} + + return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype) + + async def is_wildcard_domain(self, domain, log_info=False): + return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) + + async def handle_wildcard_event(self, event, children): + """ + Used within BBOT's scan manager to detect and tag DNS wildcard events. + + Wildcards are detected for every major record type. If a wildcard is detected, its data + is overwritten, for example: `_wildcard.evilcorp.com`. + + Args: + event (object): The event to check for wildcards. + children (list): A list of the event's resulting DNS children after resolution. + + Returns: + None: This method modifies the `event` in place and does not return a value. + + Examples: + >>> handle_wildcard_event(event, children) + # The `event` might now have tags like ["wildcard", "a-wildcard", "aaaa-wildcard"] and + # its `data` attribute might be modified to "_wildcard.evilcorp.com" if it was detected + # as a wildcard. + """ + log.debug(f"Entering handle_wildcard_event({event}, children={children})") + try: + event_host = str(event.host) + # wildcard checks + if not is_ip(event.host): + # check if the dns name itself is a wildcard entry + wildcard_rdtypes = await self.is_wildcard(event_host) + for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): + wildcard_tag = "error" + if is_wildcard == True: + event.add_tag("wildcard") + wildcard_tag = "wildcard" + event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") + + # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) + if not is_ip(event.host) and children: + if wildcard_rdtypes: + # these are the rdtypes that successfully resolve + resolved_rdtypes = set([c.upper() for c in children]) + # these are the rdtypes that have wildcards + wildcard_rdtypes_set = set(wildcard_rdtypes) + # consider the event a full wildcard if all its records are wildcards + event_is_wildcard = False + if resolved_rdtypes: + event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) + + if event_is_wildcard: + if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): + wildcard_parent = self.parent_helper.parent_domain(event_host) + for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): + if _is_wildcard: + wildcard_parent = _parent_domain + break + wildcard_data = f"_wildcard.{wildcard_parent}" + if wildcard_data != event.data: + log.debug( + f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"' + ) + event.data = wildcard_data + + # TODO: transplant this + # tag wildcard domains for convenience + # elif is_domain(event_host) or hash(event_host) in self._wildcard_cache: + # event_target = "target" in event.tags + # wildcard_domain_results = await self.is_wildcard_domain(event_host, log_info=event_target) + # for hostname, wildcard_domain_rdtypes in wildcard_domain_results.items(): + # if wildcard_domain_rdtypes: + # event.add_tag("wildcard-domain") + # for rdtype, ips in wildcard_domain_rdtypes.items(): + # event.add_tag(f"{rdtype.lower()}-wildcard-domain") + + finally: + log.debug(f"Finished handle_wildcard_event({event}, children={children})") + + async def _mock_dns(self, mock_data): + from .mock import MockResolver + + self.resolver = MockResolver(mock_data) + await self.run_and_return("_mock_dns", mock_data=mock_data) diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns/engine.py similarity index 78% rename from bbot/core/helpers/dns.py rename to bbot/core/helpers/dns/engine.py index 1ae8f0ded..54a60b328 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns/engine.py @@ -1,103 +1,68 @@ +import os import dns import time import asyncio import logging import ipaddress import traceback -import contextlib -import dns.exception -import dns.asyncresolver -from cachetools import LRUCache from contextlib import suppress +from cachetools import LRUCache -from .regexes import dns_name_regex -from bbot.core.helpers.ratelimiter import RateLimiter +from ..regexes import dns_name_regex +from bbot.core.engine import EngineServer +from bbot.core.errors import DNSWildcardBreak from bbot.core.helpers.async_helpers import NamedLock -from bbot.core.errors import ValidationError, DNSError, DNSWildcardBreak -from .misc import is_ip, is_domain, is_dns_name, domain_parents, parent_domain, rand_string, cloudcheck - -log = logging.getLogger("bbot.core.helpers.dns") - - -class BBOTAsyncResolver(dns.asyncresolver.Resolver): - """Custom asynchronous resolver for BBOT with rate limiting. - - This class extends dnspython's async resolver and provides additional support for rate-limiting DNS queries. - The maximum number of queries allowed per second can be customized via BBOT's config. - - Attributes: - _parent_helper: A reference to the instantiated `ConfigAwareHelper` (typically `scan.helpers`). - _dns_rate_limiter (RateLimiter): An instance of the RateLimiter class for DNS query rate-limiting. - - Args: - *args: Positional arguments passed to the base resolver. - **kwargs: Keyword arguments. '_parent_helper' is expected among these to provide configuration data for - rate-limiting. All other keyword arguments are passed to the base resolver. - """ - - def __init__(self, *args, **kwargs): - self._parent_helper = kwargs.pop("_parent_helper") - dns_queries_per_second = self._parent_helper.config.get("dns_queries_per_second", 100) - self._dns_rate_limiter = RateLimiter(dns_queries_per_second, "DNS") - super().__init__(*args, **kwargs) - self.rotate = True - - async def resolve(self, *args, **kwargs): - async with self._dns_rate_limiter: - return await super().resolve(*args, **kwargs) - - -class DNSHelper: - """Helper class for DNS-related operations within BBOT. - - This class provides mechanisms for host resolution, wildcard domain detection, event tagging, and more. - It centralizes all DNS-related activities in BBOT, offering both synchronous and asynchronous methods - for DNS resolution, as well as various utilities for batch resolution and DNS query filtering. - - Attributes: - parent_helper: A reference to the instantiated `ConfigAwareHelper` (typically `scan.helpers`). - resolver (BBOTAsyncResolver): An asynchronous DNS resolver tailored for BBOT with rate-limiting capabilities. - timeout (int): The timeout value for DNS queries. Defaults to 5 seconds. - retries (int): The number of retries for failed DNS queries. Defaults to 1. - abort_threshold (int): The threshold for aborting after consecutive failed queries. Defaults to 50. - max_dns_resolve_distance (int): Maximum allowed distance for DNS resolution. Defaults to 4. - all_rdtypes (list): A list of DNS record types to be considered during operations. - wildcard_ignore (tuple): Domains to be ignored during wildcard detection. - wildcard_tests (int): Number of tests to be run for wildcard detection. Defaults to 5. - _wildcard_cache (dict): Cache for wildcard detection results. - _dns_cache (LRUCache): Cache for DNS resolution results, limited in size. - _event_cache (LRUCache): Cache for event resolution results, tags. Limited in size. - resolver_file (Path): File containing system's current resolver nameservers. - filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True. - - Args: - parent_helper: The parent helper object with configuration details and utilities. - - Raises: - DNSError: If an issue arises when creating the BBOTAsyncResolver instance. - - Examples: - >>> dns_helper = DNSHelper(parent_config) - >>> resolved_host = dns_helper.resolver.resolve("example.com") - """ +from bbot.core.helpers.misc import ( + clean_dns_record, + parent_domain, + domain_parents, + is_ip, + is_domain, + is_ptr, + is_dns_name, + host_in_host, + make_ip_type, + smart_decode, + cloudcheck, + rand_string, +) + + +log = logging.getLogger("bbot.core.helpers.dns.engine.server") + + +class DNSEngine(EngineServer): + + CMDS = { + 0: "resolve", + 1: "resolve_event", + 2: "resolve_batch", + 3: "resolve_custom_batch", + 4: "is_wildcard", + 5: "is_wildcard_domain", + 99: "_mock_dns", + } all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] - def __init__(self, parent_helper): - self.parent_helper = parent_helper - try: - self.resolver = BBOTAsyncResolver(_parent_helper=self.parent_helper) - except Exception as e: - raise DNSError(f"Failed to create BBOT DNS resolver: {e}") - self.timeout = self.parent_helper.config.get("dns_timeout", 5) - self.retries = self.parent_helper.config.get("dns_retries", 1) - self.abort_threshold = self.parent_helper.config.get("dns_abort_threshold", 50) - self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 5) + def __init__(self, socket_path, config={}): + super().__init__(socket_path) + + self.config = config + # config values + self.timeout = self.config.get("dns_timeout", 5) + self.retries = self.config.get("dns_retries", 1) + self.abort_threshold = self.config.get("dns_abort_threshold", 50) + self.max_dns_resolve_distance = self.config.get("max_dns_resolve_distance", 5) + + # resolver + self.resolver = dns.asyncresolver.Resolver() + self.resolver.rotate = True self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout # skip certain queries - dns_omit_queries = self.parent_helper.config.get("dns_omit_queries", None) + dns_omit_queries = self.config.get("dns_omit_queries", None) if not dns_omit_queries: dns_omit_queries = [] self.dns_omit_queries = dict() @@ -112,36 +77,31 @@ def __init__(self, parent_helper): except KeyError: self.dns_omit_queries[rdtype] = {query} - self.wildcard_ignore = self.parent_helper.config.get("dns_wildcard_ignore", None) + # wildcard handling + self.wildcard_ignore = self.config.get("dns_wildcard_ignore", None) if not self.wildcard_ignore: self.wildcard_ignore = [] self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) - self.wildcard_tests = self.parent_helper.config.get("dns_wildcard_tests", 5) + self.wildcard_tests = self.config.get("dns_wildcard_tests", 5) self._wildcard_cache = dict() # since wildcard detection takes some time, This is to prevent multiple # modules from kicking off wildcard detection for the same domain at the same time self._wildcard_lock = NamedLock() + self._dns_connectivity_lock = asyncio.Lock() self._last_dns_success = None self._last_connectivity_warning = time.time() # keeps track of warnings issued for wildcard detection to prevent duplicate warnings self._dns_warnings = set() self._errors = dict() - self.fallback_nameservers_file = self.parent_helper.wordlist_dir / "nameservers.txt" - self._debug = self.parent_helper.config.get("dns_debug", False) - self._dummy_modules = dict() + self._debug = self.config.get("dns_debug", False) self._dns_cache = LRUCache(maxsize=10000) self._event_cache = LRUCache(maxsize=10000) self._event_cache_locks = NamedLock() - # copy the system's current resolvers to a text file for tool use - self.system_resolvers = dns.resolver.Resolver().nameservers - # TODO: DNS server speed test (start in background task) - self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False) - - self.filter_bad_ptrs = self.parent_helper.config.get("dns_filter_ptrs", True) + self.filter_bad_ptrs = self.config.get("dns_filter_ptrs", True) - async def resolve(self, query, **kwargs): + async def resolve(self, query, include_errors=False, **kwargs): """Resolve DNS names and IP addresses to their corresponding results. This is a high-level function that can translate a given domain name to its associated IP addresses @@ -163,6 +123,7 @@ async def resolve(self, query, **kwargs): {"1.2.3.4", "dead::beef"} """ results = set() + errors = [] try: r = await self.resolve_raw(query, **kwargs) if r: @@ -177,7 +138,10 @@ async def resolve(self, query, **kwargs): raise self.debug(f"Results for {query} with kwargs={kwargs}: {results}") - return results + if include_errors: + return results, errors + else: + return results async def resolve_raw(self, query, **kwargs): """Resolves the given query to its associated DNS records. @@ -272,7 +236,7 @@ async def _resolve_hostname(self, query, **kwargs): self.debug(f"Skipping {rdtype}:{query} because it's omitted in the config") return results, errors - parent = self.parent_helper.parent_domain(query) + parent = parent_domain(query) retries = kwargs.pop("retries", self.retries) use_cache = kwargs.pop("use_cache", True) tries_left = int(retries) + 1 @@ -419,8 +383,9 @@ async def handle_wildcard_event(self, event, children): log.debug(f"Entering handle_wildcard_event({event}, children={children})") try: event_host = str(event.host) + event_is_ip = is_ip(event_host) # wildcard checks - if not is_ip(event.host): + if not event_is_ip: # check if the dns name itself is a wildcard entry wildcard_rdtypes = await self.is_wildcard(event_host) for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): @@ -431,7 +396,7 @@ async def handle_wildcard_event(self, event, children): event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if not is_ip(event.host) and children: + if not event_is_ip and children: if wildcard_rdtypes: # these are the rdtypes that successfully resolve resolved_rdtypes = set([c.upper() for c in children]) @@ -444,7 +409,7 @@ async def handle_wildcard_event(self, event, children): if event_is_wildcard: if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = self.parent_helper.parent_domain(event_host) + wildcard_parent = parent_domain(event_host) for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): if _is_wildcard: wildcard_parent = _parent_domain @@ -467,7 +432,7 @@ async def handle_wildcard_event(self, event, children): finally: log.debug(f"Finished handle_wildcard_event({event}, children={children})") - async def resolve_event(self, event, minimal=False): + async def resolve_event(self, event_host, event_type, minimal=False): """ Tag the given event with the appropriate DNS record types and optionally create child events based on DNS resolutions. @@ -493,17 +458,13 @@ async def resolve_event(self, event, minimal=False): This method does not modify the passed in `event`. Instead, it returns data that can be used to modify or act upon the `event`. """ - log.debug(f"Resolving {event}") - event_host = str(event.host) + log.debug(f"Resolving event {event_type}:{event_host}") event_tags = set() dns_children = dict() event_whitelisted = False event_blacklisted = False try: - if (not event.host) or (event.type in ("IP_RANGE",)): - return event_tags, event_whitelisted, event_blacklisted, dns_children - # lock to ensure resolution of the same host doesn't start while we're working here async with self._event_cache_locks.lock(event_host): # try to get data from cache @@ -515,11 +476,11 @@ async def resolve_event(self, event, minimal=False): # then resolve types = () - if self.parent_helper.is_ip(event.host): + if is_ip(event_host): if not minimal: types = ("PTR",) else: - if event.type == "DNS_NAME" and not minimal: + if event_type == "DNS_NAME" and not minimal: types = self.all_rdtypes else: types = ("A", "AAAA") @@ -540,18 +501,19 @@ async def resolve_event(self, event, minimal=False): for r in records: for _, t in self.extract_targets(r): if t: - ip = self.parent_helper.make_ip_type(t) - - if rdtype in ("A", "AAAA", "CNAME"): - with contextlib.suppress(ValidationError): - if self.parent_helper.is_ip(ip): - if self.parent_helper.preset.whitelisted(ip): - event_whitelisted = True - with contextlib.suppress(ValidationError): - if self.parent_helper.preset.blacklisted(ip): - event_blacklisted = True - - if self.filter_bad_ptrs and rdtype in ("PTR") and self.parent_helper.is_ptr(t): + ip = make_ip_type(t) + + # TODO: transplant this + # if rdtype in ("A", "AAAA", "CNAME"): + # with contextlib.suppress(ValidationError): + # if self.parent_helper.is_ip(ip): + # if self.parent_helper.preset.whitelisted(ip): + # event_whitelisted = True + # with contextlib.suppress(ValidationError): + # if self.parent_helper.preset.blacklisted(ip): + # event_blacklisted = True + + if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(t): self.debug(f"Filtering out bad PTR: {t}") continue @@ -561,10 +523,10 @@ async def resolve_event(self, event, minimal=False): dns_children[rdtype] = {ip} # tag with cloud providers - if not self.parent_helper.in_tests: + if not self.in_tests: to_check = set() - if event.type == "IP_ADDRESS": - to_check.add(event.data) + if event_type == "IP_ADDRESS": + to_check.add(event_host) for rdtype, ips in dns_children.items(): if rdtype in ("A", "AAAA"): for ip in ips: @@ -592,7 +554,7 @@ async def resolve_event(self, event, minimal=False): return event_tags, event_whitelisted, event_blacklisted, dns_children finally: - log.debug(f"Finished resolving {event}") + log.debug(f"Finished resolving event {event_type}:{event_host}") def event_cache_get(self, host): """ @@ -648,6 +610,11 @@ async def resolve_batch(self, queries, **kwargs): for q in queries: yield (q, await self.resolve(q, **kwargs)) + async def resolve_custom_batch(self, queries): + for query, rdtype in queries: + answers, errors = await self.resolve(query, type=rdtype, include_errors=True) + yield ((query, rdtype), (answers, errors)) + def extract_targets(self, record): """ Extracts hostnames or IP addresses from a given DNS record. @@ -676,53 +643,26 @@ def extract_targets(self, record): results = set() rdtype = str(record.rdtype.name).upper() if rdtype in ("A", "AAAA", "NS", "CNAME", "PTR"): - results.add((rdtype, self._clean_dns_record(record))) + results.add((rdtype, clean_dns_record(record))) elif rdtype == "SOA": - results.add((rdtype, self._clean_dns_record(record.mname))) + results.add((rdtype, clean_dns_record(record.mname))) elif rdtype == "MX": - results.add((rdtype, self._clean_dns_record(record.exchange))) + results.add((rdtype, clean_dns_record(record.exchange))) elif rdtype == "SRV": - results.add((rdtype, self._clean_dns_record(record.target))) + results.add((rdtype, clean_dns_record(record.target))) elif rdtype == "TXT": for s in record.strings: - s = self.parent_helper.smart_decode(s) + s = smart_decode(s) for match in dns_name_regex.finditer(s): start, end = match.span() host = s[start:end] results.add((rdtype, host)) elif rdtype == "NSEC": - results.add((rdtype, self._clean_dns_record(record.next))) + results.add((rdtype, clean_dns_record(record.next))) else: log.warning(f'Unknown DNS record type "{rdtype}"') return results - @staticmethod - def _clean_dns_record(record): - """ - Cleans and formats a given DNS record for further processing. - - This static method converts the DNS record to text format if it's not already a string. - It also removes any trailing dots and converts the record to lowercase. - - Args: - record (str or dns.rdata.Rdata): The DNS record to clean. - - Returns: - str: The cleaned and formatted DNS record. - - Examples: - >>> _clean_dns_record('www.evilcorp.com.') - 'www.evilcorp.com' - - >>> from dns.rrset import from_text - >>> record = from_text('www.evilcorp.com', 3600, 'IN', 'A', '1.2.3.4')[0] - >>> _clean_dns_record(record) - '1.2.3.4' - """ - if not isinstance(record, str): - record = str(record.to_text()) - return str(record).rstrip(".").lower() - async def _catch(self, callback, *args, **kwargs): """ Asynchronously catches exceptions thrown during DNS resolution and logs them. @@ -790,55 +730,31 @@ async def is_wildcard(self, query, ips=None, rdtype=None): """ result = {} - if [ips, rdtype].count(None) == 1: - raise ValueError("Both ips and rdtype must be specified") - - if not is_dns_name(query): - return {} - - # skip check if the query's parent domain is excluded in the config - for d in self.wildcard_ignore: - if self.parent_helper.host_in_host(query, d): - log.debug(f"Skipping wildcard detection on {query} because it is excluded in the config") - return {} - - query = self._clean_dns_record(query) - # skip check if it's an IP - if is_ip(query) or not "." in query: - return {} - # skip check if the query is a domain - if is_domain(query): - return {} - parent = parent_domain(query) parents = list(domain_parents(query)) rdtypes_to_check = [rdtype] if rdtype is not None else self.all_rdtypes - base_query_ips = dict() + query_baseline = dict() # if the caller hasn't already done the work of resolving the IPs if ips is None: # then resolve the query for all rdtypes - for t in rdtypes_to_check: - raw_results, errors = await self.resolve_raw(query, type=t, use_cache=True) - if errors and not raw_results: - self.debug(f"Failed to resolve {query} ({t}) during wildcard detection") - result[t] = (None, parent) - continue - for __rdtype, answers in raw_results: - base_query_results = set() - for answer in answers: - for _, t in self.extract_targets(answer): - base_query_results.add(t) - if base_query_results: - base_query_ips[__rdtype] = base_query_results + queries = [(query, t) for t in rdtypes_to_check] + async for (query, _rdtype), (answers, errors) in self.resolve_custom_batch(queries): + if answers: + query_baseline[_rdtype] = answers + else: + if errors: + self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection") + result[_rdtype] = (None, parent) + continue else: # otherwise, we can skip all that - cleaned_ips = set([self._clean_dns_record(ip) for ip in ips]) + cleaned_ips = set([clean_dns_record(ip) for ip in ips]) if not cleaned_ips: raise ValueError("Valid IPs must be specified") - base_query_ips[rdtype] = cleaned_ips - if not base_query_ips: + query_baseline[rdtype] = cleaned_ips + if not query_baseline: return result # once we've resolved the base query and have IP addresses to work with @@ -851,9 +767,9 @@ async def is_wildcard(self, query, ips=None, rdtype=None): await self.is_wildcard_domain(host) # for every rdtype - for _rdtype in list(base_query_ips): + for _rdtype in list(query_baseline): # get the IPs from above - query_ips = base_query_ips.get(_rdtype, set()) + query_ips = query_baseline.get(_rdtype, set()) host_hash = hash(host) if host_hash in self._wildcard_cache: @@ -870,13 +786,14 @@ async def is_wildcard(self, query, ips=None, rdtype=None): result[_rdtype] = (True, host) # if we've reached a point where the dns name is a complete wildcard, class can be dismissed early - base_query_rdtypes = set(base_query_ips) + base_query_rdtypes = set(query_baseline) wildcard_rdtypes_set = set([k for k, v in result.items() if v[0] is True]) if base_query_rdtypes and wildcard_rdtypes_set and base_query_rdtypes == wildcard_rdtypes_set: log.debug( f"Breaking from wildcard detection for {query} at {host} because base query rdtypes ({base_query_rdtypes}) == wildcard rdtypes ({wildcard_rdtypes_set})" ) raise DNSWildcardBreak() + except DNSWildcardBreak: pass @@ -904,14 +821,14 @@ async def is_wildcard_domain(self, domain, log_info=False): {} """ wildcard_domain_results = {} - domain = self._clean_dns_record(domain) + domain = clean_dns_record(domain) if not is_dns_name(domain): return {} # skip check if the query's parent domain is excluded in the config for d in self.wildcard_ignore: - if self.parent_helper.host_in_host(domain, d): + if host_in_host(domain, d): log.debug(f"Skipping wildcard detection on {domain} because it is excluded in the config") return {} @@ -1010,3 +927,12 @@ def _parse_rdtype(self, t, default=None): def debug(self, *args, **kwargs): if self._debug: log.trace(*args, **kwargs) + + @property + def in_tests(self): + return os.getenv("BBOT_TESTING", "") == "True" + + async def _mock_dns(self, mock_data): + from .mock import MockResolver + + self.resolver = MockResolver(mock_data) diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py new file mode 100644 index 000000000..0685a8e80 --- /dev/null +++ b/bbot/core/helpers/dns/mock.py @@ -0,0 +1,55 @@ +import dns + +class MockResolver: + + def __init__(self, mock_data=None): + self.mock_data = mock_data if mock_data else {} + self.nameservers = ["127.0.0.1"] + + async def resolve_address(self, ipaddr, *args, **kwargs): + modified_kwargs = {} + modified_kwargs.update(kwargs) + modified_kwargs["rdtype"] = "PTR" + return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs) + + def create_dns_response(self, query_name, rdtype): + query_name = query_name.strip(".") + answers = self.mock_data.get(query_name, {}).get(rdtype, []) + if not answers: + raise self.dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}") + + message_text = f"""id 1234 +opcode QUERY +rcode NOERROR +flags QR AA RD +;QUESTION +{query_name}. IN {rdtype} +;ANSWER""" + for answer in answers: + message_text += f"\n{query_name}. 1 IN {rdtype} {answer}" + + message_text += "\n;AUTHORITY\n;ADDITIONAL\n" + message = self.dns.message.from_text(message_text) + return message + + async def resolve(self, query_name, rdtype=None): + if rdtype is None: + rdtype = "A" + elif isinstance(rdtype, str): + rdtype = rdtype.upper() + else: + rdtype = str(rdtype.name).upper() + + domain_name = self.dns.name.from_text(query_name) + rdtype_obj = self.dns.rdatatype.from_text(rdtype) + + if "_NXDOMAIN" in self.mock_data and query_name in self.mock_data["_NXDOMAIN"]: + # Simulate the NXDOMAIN exception + raise self.dns.resolver.NXDOMAIN + + try: + response = self.create_dns_response(query_name, rdtype) + answer = self.dns.resolver.Answer(domain_name, rdtype_obj, self.dns.rdataclass.IN, response) + return answer + except self.dns.resolver.NXDOMAIN: + return [] diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index c229b26a1..94fd26117 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2622,3 +2622,30 @@ async def as_completed(coros): for task in done: tasks.pop(task) yield task + + +def clean_dns_record(record): + """ + Cleans and formats a given DNS record for further processing. + + This static method converts the DNS record to text format if it's not already a string. + It also removes any trailing dots and converts the record to lowercase. + + Args: + record (str or dns.rdata.Rdata): The DNS record to clean. + + Returns: + str: The cleaned and formatted DNS record. + + Examples: + >>> clean_dns_record('www.evilcorp.com.') + 'www.evilcorp.com' + + >>> from dns.rrset import from_text + >>> record = from_text('www.evilcorp.com', 3600, 'IN', 'A', '1.2.3.4')[0] + >>> clean_dns_record(record) + '1.2.3.4' + """ + if not isinstance(record, str): + record = str(record.to_text()) + return str(record).rstrip(".").lower() diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c859d6c42..a849ae6e7 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -212,10 +212,10 @@ def __init__( self.dispatcher_tasks = [] # multiprocessing thread pool - try: - mp.set_start_method("spawn") - except Exception: - self.warning(f"Failed to set multiprocessing spawn method. This may negatively affect performance.") + start_method = mp.get_start_method() + if start_method != "spawn": + self.warning(f"Multiprocessing spawn method is set to {start_method}.") + # we spawn 1 fewer processes than cores # this helps to avoid locking up the system or competing with the main python process for cpu time num_processes = max(1, mp.cpu_count() - 1) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index d34cdf730..29c27fa93 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -1,5 +1,4 @@ import os # noqa -import dns import sys import pytest import asyncio # noqa @@ -32,6 +31,11 @@ available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal")) +@pytest.fixture(scope="session", autouse=True) +def setup_logging(): + CORE.logger.setup_queue_handler() + + @pytest.fixture def clean_default_config(monkeypatch): clean_config = OmegaConf.merge( @@ -216,67 +220,3 @@ def install_all_python_deps(): for module in DEFAULT_PRESET.module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) - - -class MockResolver: - import dns - - def __init__(self, mock_data=None): - self.mock_data = mock_data if mock_data else {} - self.nameservers = ["127.0.0.1"] - - async def resolve_address(self, ipaddr, *args, **kwargs): - modified_kwargs = {} - modified_kwargs.update(kwargs) - modified_kwargs["rdtype"] = "PTR" - return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs) - - def create_dns_response(self, query_name, rdtype): - query_name = query_name.strip(".") - answers = self.mock_data.get(query_name, {}).get(rdtype, []) - if not answers: - raise self.dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}") - - message_text = f"""id 1234 -opcode QUERY -rcode NOERROR -flags QR AA RD -;QUESTION -{query_name}. IN {rdtype} -;ANSWER""" - for answer in answers: - message_text += f"\n{query_name}. 1 IN {rdtype} {answer}" - - message_text += "\n;AUTHORITY\n;ADDITIONAL\n" - message = self.dns.message.from_text(message_text) - return message - - async def resolve(self, query_name, rdtype=None): - if rdtype is None: - rdtype = "A" - elif isinstance(rdtype, str): - rdtype = rdtype.upper() - else: - rdtype = str(rdtype.name).upper() - - domain_name = self.dns.name.from_text(query_name) - rdtype_obj = self.dns.rdatatype.from_text(rdtype) - - if "_NXDOMAIN" in self.mock_data and query_name in self.mock_data["_NXDOMAIN"]: - # Simulate the NXDOMAIN exception - raise self.dns.resolver.NXDOMAIN - - try: - response = self.create_dns_response(query_name, rdtype) - answer = self.dns.resolver.Answer(domain_name, rdtype_obj, self.dns.rdataclass.IN, response) - return answer - except self.dns.resolver.NXDOMAIN: - return [] - - -@pytest.fixture() -def mock_dns(): - def _mock_dns(scan, mock_data): - scan.helpers.dns.resolver = MockResolver(mock_data) - - return _mock_dns diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 6fd51800f..594192568 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -2,72 +2,110 @@ @pytest.mark.asyncio -async def test_dns(bbot_scanner, mock_dns): +async def test_dns_engine(bbot_scanner): + scan = bbot_scanner() + result = await scan.helpers.resolve("one.one.one.one") + assert "1.1.1.1" in result + assert "2606:4700:4700::1111" in result + + results = [_ async for _ in scan.helpers.resolve_batch(("one.one.one.one", "1.1.1.1"))] + pass_1 = False + pass_2 = False + for query, result in results: + if query == "one.one.one.one" and "1.1.1.1" in result: + pass_1 = True + elif query == "1.1.1.1" and "one.one.one.one" in result: + pass_2 = True + assert pass_1 and pass_2 + + results = [_ async for _ in scan.helpers.resolve_custom_batch((("one.one.one.one", "A"), ("1.1.1.1", "PTR")))] + pass_1 = False + pass_2 = False + for (query, rdtype), (result, errors) in results: + if query == "one.one.one.one" and "1.1.1.1" in result: + pass_1 = True + elif query == "1.1.1.1" and "one.one.one.one" in result: + pass_2 = True + assert pass_1 and pass_2 + + +@pytest.mark.asyncio +async def test_dns(bbot_scanner): scan = bbot_scanner("1.1.1.1") - helpers = scan.helpers + + from bbot.core.helpers.dns.engine import DNSEngine + + dnsengine = DNSEngine(None) # lowest level functions - a_responses = await helpers._resolve_hostname("one.one.one.one") - aaaa_responses = await helpers._resolve_hostname("one.one.one.one", rdtype="AAAA") - ip_responses = await helpers._resolve_ip("1.1.1.1") + a_responses = await dnsengine._resolve_hostname("one.one.one.one") + aaaa_responses = await dnsengine._resolve_hostname("one.one.one.one", rdtype="AAAA") + ip_responses = await dnsengine._resolve_ip("1.1.1.1") assert a_responses[0].response.answer[0][0].address in ("1.1.1.1", "1.0.0.1") assert aaaa_responses[0].response.answer[0][0].address in ("2606:4700:4700::1111", "2606:4700:4700::1001") assert ip_responses[0].response.answer[0][0].target.to_text() in ("one.one.one.one.",) # mid level functions - _responses, errors = await helpers.resolve_raw("one.one.one.one") + _responses, errors = await dnsengine.resolve_raw("one.one.one.one") responses = [] for rdtype, response in _responses: for answers in response: - responses += list(helpers.extract_targets(answers)) + responses += list(dnsengine.extract_targets(answers)) assert ("A", "1.1.1.1") in responses - _responses, errors = await helpers.resolve_raw("one.one.one.one", rdtype="AAAA") + _responses, errors = await dnsengine.resolve_raw("one.one.one.one", rdtype="AAAA") responses = [] for rdtype, response in _responses: for answers in response: - responses += list(helpers.extract_targets(answers)) + responses += list(dnsengine.extract_targets(answers)) assert ("AAAA", "2606:4700:4700::1111") in responses - _responses, errors = await helpers.resolve_raw("1.1.1.1") + _responses, errors = await dnsengine.resolve_raw("1.1.1.1") responses = [] for rdtype, response in _responses: for answers in response: - responses += list(helpers.extract_targets(answers)) + responses += list(dnsengine.extract_targets(answers)) assert ("PTR", "one.one.one.one") in responses # high level functions - assert "1.1.1.1" in await helpers.resolve("one.one.one.one") - assert "2606:4700:4700::1111" in await helpers.resolve("one.one.one.one", type="AAAA") - assert "one.one.one.one" in await helpers.resolve("1.1.1.1") + assert "1.1.1.1" in await dnsengine.resolve("one.one.one.one") + assert "2606:4700:4700::1111" in await dnsengine.resolve("one.one.one.one", type="AAAA") + assert "one.one.one.one" in await dnsengine.resolve("1.1.1.1") for rdtype in ("NS", "SOA", "MX", "TXT"): - assert len(await helpers.resolve("google.com", type=rdtype)) > 0 + assert len(await dnsengine.resolve("google.com", type=rdtype)) > 0 # batch resolution - batch_results = [r async for r in helpers.resolve_batch(["1.1.1.1", "one.one.one.one"])] + batch_results = [r async for r in dnsengine.resolve_batch(["1.1.1.1", "one.one.one.one"])] assert len(batch_results) == 2 batch_results = dict(batch_results) assert any([x in batch_results["one.one.one.one"] for x in ("1.1.1.1", "1.0.0.1")]) assert "one.one.one.one" in batch_results["1.1.1.1"] + # custom batch resolution + batch_results = [r async for r in dnsengine.resolve_custom_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])] + assert len(batch_results) == 2 + batch_results = dict(batch_results) + assert any([x in batch_results[("one.one.one.one", "A")][0] for x in ("1.1.1.1", "1.0.0.1")]) + assert "one.one.one.one" in batch_results[("1.1.1.1", "PTR")][0] + # "any" type - resolved = await helpers.resolve("google.com", type="any") - assert any([helpers.is_subdomain(h) for h in resolved]) + resolved = await dnsengine.resolve("google.com", type="any") + assert any([scan.helpers.is_subdomain(h) for h in resolved]) # dns cache - helpers.dns._dns_cache.clear() - assert hash(f"1.1.1.1:PTR") not in helpers.dns._dns_cache - assert hash(f"one.one.one.one:A") not in helpers.dns._dns_cache - assert hash(f"one.one.one.one:AAAA") not in helpers.dns._dns_cache - await helpers.resolve("1.1.1.1", use_cache=False) - await helpers.resolve("one.one.one.one", use_cache=False) - assert hash(f"1.1.1.1:PTR") not in helpers.dns._dns_cache - assert hash(f"one.one.one.one:A") not in helpers.dns._dns_cache - assert hash(f"one.one.one.one:AAAA") not in helpers.dns._dns_cache - - await helpers.resolve("1.1.1.1") - assert hash(f"1.1.1.1:PTR") in helpers.dns._dns_cache - await helpers.resolve("one.one.one.one") - assert hash(f"one.one.one.one:A") in helpers.dns._dns_cache - assert hash(f"one.one.one.one:AAAA") in helpers.dns._dns_cache + dnsengine._dns_cache.clear() + assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache + assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache + assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache + await dnsengine.resolve("1.1.1.1", use_cache=False) + await dnsengine.resolve("one.one.one.one", use_cache=False) + assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache + assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache + assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache + + await dnsengine.resolve("1.1.1.1") + assert hash(f"1.1.1.1:PTR") in dnsengine._dns_cache + await dnsengine.resolve("one.one.one.one") + assert hash(f"one.one.one.one:A") in dnsengine._dns_cache + assert hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache # Ensure events with hosts have resolved_hosts attribute populated resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", dummy=True) @@ -83,8 +121,7 @@ async def test_dns(bbot_scanner, mock_dns): assert set(children1.keys()) == set(children2.keys()) scan2 = bbot_scanner("evilcorp.com", config={"dns_resolution": True}) - mock_dns( - scan2, + await scan2.helpers.dns._mock_dns( { "evilcorp.com": {"TXT": ['"v=spf1 include:cloudprovider.com ~all"']}, "cloudprovider.com": {"A": ["1.2.3.4"]}, @@ -101,33 +138,37 @@ async def test_wildcards(bbot_scanner): scan = bbot_scanner("1.1.1.1") helpers = scan.helpers + from bbot.core.helpers.dns.engine import DNSEngine + + dnsengine = DNSEngine(None) + # wildcards - wildcard_domains = await helpers.is_wildcard_domain("asdf.github.io") - assert hash("github.io") in helpers.dns._wildcard_cache - assert hash("asdf.github.io") in helpers.dns._wildcard_cache + wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io") + assert hash("github.io") in dnsengine._wildcard_cache + assert hash("asdf.github.io") in dnsengine._wildcard_cache assert "github.io" in wildcard_domains assert "A" in wildcard_domains["github.io"] assert "SRV" not in wildcard_domains["github.io"] assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"]) - helpers.dns._wildcard_cache.clear() + dnsengine._wildcard_cache.clear() - wildcard_rdtypes = await helpers.is_wildcard("blacklanternsecurity.github.io") + wildcard_rdtypes = await dnsengine.is_wildcard("blacklanternsecurity.github.io") assert "A" in wildcard_rdtypes assert "SRV" not in wildcard_rdtypes assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in helpers.dns._wildcard_cache - assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 - helpers.dns._wildcard_cache.clear() + assert hash("github.io") in dnsengine._wildcard_cache + assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0 + dnsengine._wildcard_cache.clear() - wildcard_rdtypes = await helpers.is_wildcard("asdf.asdf.asdf.github.io") + wildcard_rdtypes = await dnsengine.is_wildcard("asdf.asdf.asdf.github.io") assert "A" in wildcard_rdtypes assert "SRV" not in wildcard_rdtypes assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.asdf.github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.asdf.asdf.github.io") in helpers.dns._wildcard_cache - assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 + assert hash("github.io") in dnsengine._wildcard_cache + assert not hash("asdf.github.io") in dnsengine._wildcard_cache + assert not hash("asdf.asdf.github.io") in dnsengine._wildcard_cache + assert not hash("asdf.asdf.asdf.github.io") in dnsengine._wildcard_cache + assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0 wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", dummy=True) wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", dummy=True) wildcard_event3 = scan.make_event("github.io", "DNS_NAME", dummy=True) @@ -147,6 +188,7 @@ async def test_wildcards(bbot_scanner): assert "srv-wildcard" not in wildcard_event2.tags assert wildcard_event1.data == "_wildcard.github.io" assert wildcard_event2.data == "_wildcard.github.io" - assert "wildcard-domain" in wildcard_event3.tags - assert "a-wildcard-domain" in wildcard_event3.tags - assert "srv-wildcard-domain" not in wildcard_event3.tags + # TODO: re-enable this? + # assert "wildcard-domain" in wildcard_event3.tags + # assert "a-wildcard-domain" in wildcard_event3.tags + # assert "srv-wildcard-domain" not in wildcard_event3.tags diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 435e57991..63305d0e4 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -61,7 +61,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) scan.modules["per_hostport_only"] = per_hostport_only scan.modules["per_domain_only"] = per_domain_only if _dns_mock: - mock_dns(scan, _dns_mock) + await scan.helpers.dns._mock_dns(_dns_mock) if scan_callback is not None: scan_callback(scan) return ( diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 3b9f57767..f3b589d7c 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -799,7 +799,7 @@ async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog, mock_dns whitelist=["127.0.0.0/29", "test.notreal"], blacklist=["127.0.0.64/29"], ) - mock_dns(scan, { + await scan.helpers.dns._mock_dns({ "www-prod.test.notreal": {"A": ["127.0.0.66"]}, "www-dev.test.notreal": {"A": ["127.0.0.22"]}, }) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 2fada7968..7b2997f65 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -326,8 +326,7 @@ async def handle_event(self, event): output_modules=["python"], force_start=True, ) - mock_dns( - scan, + await scan.helpers.dns._mock_dns( { "evilcorp.com": {"A": ["127.0.254.1"]}, "www.evilcorp.com": {"A": ["127.0.254.2"]}, diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 3f1c01c04..ace0cad7b 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -59,7 +59,7 @@ async def test_scan( # make sure DNS resolution works scan4 = bbot_scanner("1.1.1.1", config={"dns_resolution": True}) - mock_dns(scan4, dns_table) + await scan4.helpers.dns._mock_dns(dns_table) events = [] async for event in scan4.async_start(): events.append(event) @@ -68,7 +68,7 @@ async def test_scan( # make sure it doesn't work when you turn it off scan5 = bbot_scanner("1.1.1.1", config={"dns_resolution": False}) - mock_dns(scan5, dns_table) + await scan5.helpers.dns._mock_dns(dns_table) events = [] async for event in scan5.async_start(): events.append(event) diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index a4ea06f81..8db36cd8d 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -91,10 +91,10 @@ def set_expect_requests(self, expect_args={}, respond_args={}): def set_expect_requests_handler(self, expect_args=None, request_handler=None): self.httpserver.expect_request(expect_args).respond_with_handler(request_handler) - def mock_dns(self, mock_data, scan=None): + async def mock_dns(self, mock_data, scan=None): if scan is None: scan = self.scan - scan.helpers.dns.resolver = MockResolver(mock_data) + await scan.helpers.dns._dns_mock(mock_data) @property def module(self): diff --git a/bbot/test/test_step_2/module_tests/test_module_affiliates.py b/bbot/test/test_step_2/module_tests/test_module_affiliates.py index 4afd4cd29..b138dce65 100644 --- a/bbot/test/test_step_2/module_tests/test_module_affiliates.py +++ b/bbot/test/test_step_2/module_tests/test_module_affiliates.py @@ -6,7 +6,7 @@ class TestAffiliates(ModuleTestBase): config_overrides = {"dns_resolution": True} async def setup_before_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( { "8.8.8.8.in-addr.arpa": {"PTR": ["dns.google"]}, "dns.google": {"A": ["8.8.8.8"], "NS": ["ns1.zdns.google"]}, diff --git a/bbot/test/test_step_2/module_tests/test_module_aggregate.py b/bbot/test/test_step_2/module_tests/test_module_aggregate.py index 7a41fe022..a41c60701 100644 --- a/bbot/test/test_step_2/module_tests/test_module_aggregate.py +++ b/bbot/test/test_step_2/module_tests/test_module_aggregate.py @@ -5,7 +5,7 @@ class TestAggregate(ModuleTestBase): config_overrides = {"dns_resolution": True, "scope_report_distance": 1} async def setup_before_prep(self, module_test): - module_test.mock_dns({"blacklanternsecurity.com": {"A": ["1.2.3.4"]}}) + await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["1.2.3.4"]}}) def check(self, module_test, events): filename = next(module_test.scan.home.glob("scan-stats-table*.txt")) diff --git a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py index af46ad5ba..6b6c78dbf 100644 --- a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py +++ b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py @@ -8,7 +8,7 @@ class TestAsset_Inventory(ModuleTestBase): modules_overrides = ["asset_inventory", "nmap", "sslcert"] async def setup_before_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( { "1.0.0.127.in-addr.arpa": {"PTR": ["www.bbottest.notreal"]}, "www.bbottest.notreal": {"A": ["127.0.0.1"]}, diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns.py b/bbot/test/test_step_2/module_tests/test_module_baddns.py index 57cca7d5f..dd533669f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns.py @@ -24,7 +24,7 @@ async def setup_after_prep(self, module_test): from bbot.modules import baddns as baddns_module from baddns.lib.whoismanager import WhoisManager - module_test.mock_dns( + await module_test.mock_dns( {"bad.dns": {"CNAME": ["baddns.azurewebsites.net."]}, "_NXDOMAIN": ["baddns.azurewebsites.net"]} ) module_test.monkeypatch.setattr(baddns_module.baddns, "select_modules", self.select_modules) @@ -52,7 +52,7 @@ def set_target(self, target): respond_args = {"response_data": "

Oops! We couldn’t find that page.

", "status": 200} module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - module_test.mock_dns( + await module_test.mock_dns( {"bad.dns": {"CNAME": ["baddns.bigcartel.com."]}, "baddns.bigcartel.com": {"A": ["127.0.0.1"]}} ) module_test.monkeypatch.setattr(baddns_module.baddns, "select_modules", self.select_modules) diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py index 349db8ed8..b3810e75a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py @@ -31,7 +31,7 @@ def from_xfr(*args, **kwargs): zone = dns.zone.from_text(zone_text, origin="bad.dns.") return zone - module_test.mock_dns({"bad.dns": {"NS": ["ns1.bad.dns."]}, "ns1.bad.dns": {"A": ["127.0.0.1"]}}) + await module_test.mock_dns({"bad.dns": {"NS": ["ns1.bad.dns."]}, "ns1.bad.dns": {"A": ["127.0.0.1"]}}) module_test.monkeypatch.setattr("dns.zone.from_xfr", from_xfr) module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS) @@ -46,7 +46,7 @@ class TestBaddns_zone_nsec(BaseTestBaddns_zone): async def setup_after_prep(self, module_test): from baddns.lib.whoismanager import WhoisManager - module_test.mock_dns( + await module_test.mock_dns( { "bad.dns": {"NSEC": ["asdf.bad.dns"]}, "asdf.bad.dns": {"NSEC": ["zzzz.bad.dns"]}, diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py index 34c73de82..73260f327 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -48,7 +48,7 @@ async def setup_before_prep(self, module_test): url=f"https://api.dehashed.com/search?query=domain:blacklanternsecurity.com&size=10000&page=1", json=dehashed_domain_response, ) - module_test.mock_dns( + await module_test.mock_dns( { "bob.com": {"A": ["127.0.0.1"]}, "blacklanternsecurity.com": {"A": ["127.0.0.1"]}, diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index aaf26664c..5850fbd49 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -6,7 +6,7 @@ class TestDNSCommonSRV(ModuleTestBase): config_overrides = {"dns_resolution": True} async def setup_after_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( { "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal": { "SRV": ["0 100 3268 asdf.blacklanternsecurity.notreal"] diff --git a/bbot/test/test_step_2/module_tests/test_module_internetdb.py b/bbot/test/test_step_2/module_tests/test_module_internetdb.py index d24cdebc0..54ec6a163 100644 --- a/bbot/test/test_step_2/module_tests/test_module_internetdb.py +++ b/bbot/test/test_step_2/module_tests/test_module_internetdb.py @@ -5,7 +5,7 @@ class TestInternetDB(ModuleTestBase): config_overrides = {"dns_resolution": True} async def setup_before_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( { "blacklanternsecurity.com": {"A": ["1.2.3.4"]}, "autodiscover.blacklanternsecurity.com": {"A": ["2.3.4.5"]}, diff --git a/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py b/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py index b8ba8331a..edb7dbff6 100644 --- a/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py +++ b/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py @@ -6,7 +6,7 @@ class TestIPNeighbor(ModuleTestBase): config_overrides = {"scope_report_distance": 1, "dns_resolution": True, "scope_dns_search_distance": 2} async def setup_after_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( {"3.0.0.127.in-addr.arpa": {"PTR": ["asdf.www.bls.notreal"]}, "asdf.www.bls.notreal": {"A": ["127.0.0.3"]}} ) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index 21f464054..8e9c0f3bf 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -235,7 +235,7 @@ async def new_emit_event(event_data, event_type, **kwargs): await old_emit_event(event_data, event_type, **kwargs) module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event) - module_test.mock_dns({"asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}}) + await module_test.mock_dns({"asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}}) request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862") respond_args = dict(response_data="https://asdf.blacklanternsecurity.com") diff --git a/bbot/test/test_step_2/module_tests/test_module_speculate.py b/bbot/test/test_step_2/module_tests/test_module_speculate.py index 2dcafaddc..2f7d6b7f3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_speculate.py +++ b/bbot/test/test_step_2/module_tests/test_module_speculate.py @@ -28,7 +28,7 @@ class TestSpeculate_OpenPorts(ModuleTestBase): config_overrides = {"speculate": True} async def setup_before_prep(self, module_test): - module_test.mock_dns( + await module_test.mock_dns( { "evilcorp.com": {"A": ["127.0.254.1"]}, "asdf.evilcorp.com": {"A": ["127.0.254.2"]}, From 2649425f80fb54d1e91f93b59193bf33f6cf513a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 12:22:39 -0400 Subject: [PATCH 087/171] blacked --- bbot/core/helpers/dns/mock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py index 0685a8e80..de283d517 100644 --- a/bbot/core/helpers/dns/mock.py +++ b/bbot/core/helpers/dns/mock.py @@ -1,5 +1,6 @@ import dns + class MockResolver: def __init__(self, mock_data=None): From ec2f89a67ec8d6540e689702ab620e9df9ce32ab Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 12:27:21 -0400 Subject: [PATCH 088/171] add pyzmq dependency --- poetry.lock | 107 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index a12258088..818842c3a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1901,6 +1901,111 @@ files = [ [package.dependencies] pyyaml = "*" +[[package]] +name = "pyzmq" +version = "25.1.2" +description = "Python bindings for 0MQ" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, + {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, + {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, + {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, + {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, + {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, + {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, + {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, + {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, + {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, + {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, + {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, + {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, +] + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} + [[package]] name = "regex" version = "2023.12.25" @@ -2431,4 +2536,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "30069bdb734215feeee60b9bbafeb5c7c8708044ea92c25465370dbad31d808f" +content-hash = "07cc1440e8eda2b5dc94d6cff919e332e36ea728fe786f217f17a25a64922683" diff --git a/pyproject.toml b/pyproject.toml index ecd8c18d6..70c839795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ tldextract = "^5.1.1" cachetools = "^5.3.2" socksio = "^1.0.0" jinja2 = "^3.1.3" +pyzmq = "^25.1.2" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From 42232d020ce64b8114420896cdfdf6b2e066fb17 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 13:01:20 -0400 Subject: [PATCH 089/171] fix dns mocker --- bbot/core/config/logger.py | 6 +++++- bbot/core/engine.py | 2 +- bbot/core/helpers/dns/engine.py | 1 + bbot/core/helpers/dns/mock.py | 14 +++++++------- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 3844a65fc..276993ead 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -59,6 +59,7 @@ def __init__(self, core): self._loggers = None self._log_handlers = None self._log_level = None + self.root_logger = logging.getLogger() self.core_logger = logging.getLogger("bbot") self.core = core @@ -79,7 +80,10 @@ def setup_queue_handler(self, logging_queue=None): else: self.queue = logging_queue self.queue_handler = logging.handlers.QueueHandler(logging_queue) - logging.getLogger().addHandler(self.queue_handler) + + if self.queue_handler not in self.root_logger.handlers: + self.root_logger.addHandler(self.queue_handler) + self.core_logger.setLevel(self.log_level) # disable asyncio logging for child processes if self.process_name != "MainProcess": diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 5f4530ba1..cf9b90f28 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -155,7 +155,7 @@ async def worker(self): try: while 1: client_id, binary = await self.socket.recv_multipart() - self.log.debug(f"{self.name} got binary: {binary}") + # self.log.debug(f"{self.name} got binary: {binary}") message = pickle.loads(binary) self.log.debug(f"{self.name} got message: {message}") diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 54a60b328..470311b48 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -933,6 +933,7 @@ def in_tests(self): return os.getenv("BBOT_TESTING", "") == "True" async def _mock_dns(self, mock_data): + self.log.critical(f"SETTING MOCK RESOLVER") from .mock import MockResolver self.resolver = MockResolver(mock_data) diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py index de283d517..70d978aff 100644 --- a/bbot/core/helpers/dns/mock.py +++ b/bbot/core/helpers/dns/mock.py @@ -17,7 +17,7 @@ def create_dns_response(self, query_name, rdtype): query_name = query_name.strip(".") answers = self.mock_data.get(query_name, {}).get(rdtype, []) if not answers: - raise self.dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}") + raise dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}") message_text = f"""id 1234 opcode QUERY @@ -30,7 +30,7 @@ def create_dns_response(self, query_name, rdtype): message_text += f"\n{query_name}. 1 IN {rdtype} {answer}" message_text += "\n;AUTHORITY\n;ADDITIONAL\n" - message = self.dns.message.from_text(message_text) + message = dns.message.from_text(message_text) return message async def resolve(self, query_name, rdtype=None): @@ -41,16 +41,16 @@ async def resolve(self, query_name, rdtype=None): else: rdtype = str(rdtype.name).upper() - domain_name = self.dns.name.from_text(query_name) - rdtype_obj = self.dns.rdatatype.from_text(rdtype) + domain_name = dns.name.from_text(query_name) + rdtype_obj = dns.rdatatype.from_text(rdtype) if "_NXDOMAIN" in self.mock_data and query_name in self.mock_data["_NXDOMAIN"]: # Simulate the NXDOMAIN exception - raise self.dns.resolver.NXDOMAIN + raise dns.resolver.NXDOMAIN try: response = self.create_dns_response(query_name, rdtype) - answer = self.dns.resolver.Answer(domain_name, rdtype_obj, self.dns.rdataclass.IN, response) + answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response) return answer - except self.dns.resolver.NXDOMAIN: + except dns.resolver.NXDOMAIN: return [] From 2a1941797118c5701522aab499bce43d0c9476c8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 13:33:07 -0400 Subject: [PATCH 090/171] fix test_event --- bbot/test/bbot_fixtures.py | 6 ------ bbot/test/test_step_1/test_modules_basic.py | 5 ----- 2 files changed, 11 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 29c27fa93..4452cc28f 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -74,12 +74,6 @@ def scan(monkeypatch): from bbot.scanner import Scanner bbot_scan = Scanner("127.0.0.1", modules=["ipneighbor"]) - - fallback_nameservers_file = bbot_scan.helpers.bbot_home / "fallback_nameservers.txt" - with open(fallback_nameservers_file, "w") as f: - f.write("8.8.8.8\n") - monkeypatch.setattr(bbot_scan.helpers.dns, "fallback_nameservers_file", fallback_nameservers_file) - return bbot_scan diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 7b2997f65..b1891ce05 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -10,10 +10,6 @@ @pytest.mark.asyncio async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock): - fallback_nameservers = scan.helpers.temp_dir / "nameservers.txt" - with open(fallback_nameservers, "w") as f: - f.write("8.8.8.8\n") - for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"): httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"}) @@ -85,7 +81,6 @@ async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock): config={i: True for i in available_internal_modules}, force_start=True, ) - scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers await scan2.load_modules() scan2.status = "RUNNING" From cd015a6aca3f6892d839a444934a4d3dae0aae9c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 13:41:33 -0400 Subject: [PATCH 091/171] working on tests --- .../test_step_1/test_manager_deduplication.py | 26 +------------------ .../test_manager_scope_accuracy.py | 6 ++--- bbot/test/test_step_1/test_scan.py | 1 - 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 63305d0e4..d3221d554 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -3,7 +3,7 @@ @pytest.mark.asyncio -async def test_manager_deduplication(bbot_scanner, mock_dns): +async def test_manager_deduplication(bbot_scanner): class DefaultModule(BaseModule): _name = "default_module" @@ -90,30 +90,6 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - # SCAN / severe_jacqueline (SCAN:d233093d39044c961754c97f749fa758543d7474) - # DNS_NAME / test.notreal - # OPEN_TCP_PORT / default_module.test.notreal:88 - # OPEN_TCP_PORT / test.notreal:88 - # DNS_NAME / default_module.test.notreal - # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 - # OPEN_TCP_PORT / accept_dupes.test.notreal:88 - # DNS_NAME / per_hostport_only.test.notreal - # DNS_NAME / no_suppress_dupes.test.notreal - # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 - # DNS_NAME / no_suppress_dupes.test.notreal - # DNS_NAME / no_suppress_dupes.test.notreal - # DNS_NAME / accept_dupes.test.notreal - # OPEN_TCP_PORT / per_domain_only.test.notreal:88 - # DNS_NAME / no_suppress_dupes.test.notreal - # DNS_NAME / no_suppress_dupes.test.notreal - # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 - # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 - # DNS_NAME / per_domain_only.test.notreal - # OPEN_TCP_PORT / per_hostport_only.test.notreal:88 - # OPEN_TCP_PORT / no_suppress_dupes.test.notreal:88 - - for e in events: - log.critical(f"{e.type} / {e.data} / {e.module} / {e.source.data} / {e.source.module}") assert len(events) == 21 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index f3b589d7c..f111dbf94 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -31,7 +31,7 @@ def bbot_other_httpservers(): @pytest.mark.asyncio -async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_httpservers, bbot_httpserver_ssl, mock_dns): +async def test_manager_scope_accuracy(bbot_scanner, bbot_httpserver, bbot_other_httpservers, bbot_httpserver_ssl): """ This test ensures that BBOT correctly handles different scope distance settings. It performs these tests for normal modules, output modules, and their graph variants, @@ -101,7 +101,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) scan.modules["dummy_graph_output_module"] = dummy_graph_output_module scan.modules["dummy_graph_batch_output_module"] = dummy_graph_batch_output_module if _dns_mock: - mock_dns(scan, _dns_mock) + await scan.helpers.dns._mock_dns(_dns_mock) if scan_callback is not None: scan_callback(scan) return ( @@ -787,7 +787,7 @@ def custom_setup(scan): @pytest.mark.asyncio -async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog, mock_dns): +async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog): bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="") diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index ace0cad7b..e5648c4ee 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -7,7 +7,6 @@ async def test_scan( helpers, monkeypatch, bbot_scanner, - mock_dns, ): scan0 = bbot_scanner( "1.1.1.1/31", From 45f8f1a3063b9bbc4347bf13d76e693d5c20b6e0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 14:22:02 -0400 Subject: [PATCH 092/171] more work on tests --- bbot/core/helpers/dns/dns.py | 24 ++++++++++- bbot/core/helpers/dns/engine.py | 42 +++++++------------ .../test_manager_scope_accuracy.py | 1 + 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 1a9474277..dd1d61e75 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -2,8 +2,10 @@ import logging import dns.exception import dns.asyncresolver +from contextlib import suppress from bbot.core.engine import EngineClient +from bbot.core.errors import ValidationError from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host from .engine import DNSEngine @@ -93,7 +95,27 @@ async def resolve_event(self, event, minimal=False): event_host = str(event.host) event_type = str(event.type) kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} - return await self.run_and_return("resolve_event", **kwargs) + event_tags, dns_children = await self.run_and_return("resolve_event", **kwargs) + + # whitelisting / blacklisting based on resolved hosts + event_whitelisted = False + event_blacklisted = False + for rdtype, children in dns_children.items(): + for host in children: + if rdtype in ("A", "AAAA", "CNAME"): + # having a CNAME to an in-scope resource doesn't make you in-scope + if rdtype != "CNAME": + with suppress(ValidationError): + if self.parent_helper.scan.whitelisted(host): + self.log.critical(f"{event_host} --> {host} is whitelisted") + event_whitelisted = True + # CNAME to a blacklisted resources, means you're blacklisted + with suppress(ValidationError): + if self.parent_helper.scan.blacklisted(host): + self.log.critical(f"{event_host} --> {host} is blacklisted") + event_blacklisted = True + + return event_tags, event_whitelisted, event_blacklisted, dns_children async def is_wildcard(self, query, ips=None, rdtype=None): """ diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 470311b48..8c6d26eb4 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -445,8 +445,6 @@ async def resolve_event(self, event_host, event_type, minimal=False): Returns: tuple: A 4-tuple containing the following items: - event_tags (set): Set of tags for the event. - - event_whitelisted (bool): Whether the event is whitelisted. - - event_blacklisted (bool): Whether the event is blacklisted. - dns_children (dict): Dictionary containing child events from DNS resolutions. Examples: @@ -461,18 +459,19 @@ async def resolve_event(self, event_host, event_type, minimal=False): log.debug(f"Resolving event {event_type}:{event_host}") event_tags = set() dns_children = dict() - event_whitelisted = False - event_blacklisted = False try: # lock to ensure resolution of the same host doesn't start while we're working here async with self._event_cache_locks.lock(event_host): # try to get data from cache - _event_tags, _event_whitelisted, _event_blacklisted, _dns_children = self.event_cache_get(event_host) - event_tags.update(_event_tags) - # if we found it, return it - if _event_whitelisted is not None: - return event_tags, _event_whitelisted, _event_blacklisted, _dns_children + try: + _event_tags, _dns_children = self._event_cache[event_host] + event_tags.update(_event_tags) + # if we found it, return it + if _event_tags is not None: + return event_tags, _dns_children + except KeyError: + _event_tags, _dns_children = set(), set() # then resolve types = () @@ -497,22 +496,11 @@ async def resolve_event(self, event_host, event_type, minimal=False): event_tags.add("resolved") event_tags.add(f"{rdtype.lower()}-record") - # whitelisting and blacklisting of IPs for r in records: for _, t in self.extract_targets(r): if t: ip = make_ip_type(t) - # TODO: transplant this - # if rdtype in ("A", "AAAA", "CNAME"): - # with contextlib.suppress(ValidationError): - # if self.parent_helper.is_ip(ip): - # if self.parent_helper.preset.whitelisted(ip): - # event_whitelisted = True - # with contextlib.suppress(ValidationError): - # if self.parent_helper.preset.blacklisted(ip): - # event_blacklisted = True - if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(t): self.debug(f"Filtering out bad PTR: {t}") continue @@ -549,9 +537,9 @@ async def resolve_event(self, event_host, event_type, minimal=False): except ValueError: continue - self._event_cache[event_host] = (event_tags, event_whitelisted, event_blacklisted, dns_children) + self._event_cache[event_host] = (event_tags, dns_children) - return event_tags, event_whitelisted, event_blacklisted, dns_children + return event_tags, dns_children finally: log.debug(f"Finished resolving event {event_type}:{event_host}") @@ -566,8 +554,6 @@ def event_cache_get(self, host): Returns: tuple: A 4-tuple containing the following items: - event_tags (set): Set of tags for the event. - - event_whitelisted (bool or None): Whether the event is whitelisted. Returns None if not found. - - event_blacklisted (bool or None): Whether the event is blacklisted. Returns None if not found. - dns_children (set): Set containing child events from DNS resolutions. Examples: @@ -579,13 +565,13 @@ def event_cache_get(self, host): Assuming no event with host "www.notincache.com" has been cached: >>> event_cache_get("www.notincache.com") - (set(), None, None, set()) + (set(), set()) """ try: - event_tags, event_whitelisted, event_blacklisted, dns_children = self._event_cache[host] - return (event_tags, event_whitelisted, event_blacklisted, dns_children) + event_tags, dns_children = self._event_cache[host] + return (event_tags, dns_children) except KeyError: - return set(), None, None, set() + return set(), set() async def resolve_batch(self, queries, **kwargs): """ diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index f111dbf94..d1c17f185 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -807,6 +807,7 @@ async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog): events = [e async for e in scan.async_start()] assert any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-dev.test.notreal:8888/"]) + # the hostname is in-scope, but its IP is blacklisted, therefore we shouldn't see it assert not any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-prod.test.notreal:8888/"]) assert 'Omitting due to blacklisted DNS associations: URL_UNVERIFIED("http://www-prod.test.notreal:8888/"' in caplog.text From 98bbb73bcecb5f1055eabf0387d85048b7bb01f6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 14:49:10 -0400 Subject: [PATCH 093/171] still working on tests --- bbot/core/helpers/dns/dns.py | 7 ++++--- bbot/core/helpers/dns/engine.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index dd1d61e75..3bcd92b10 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -101,19 +101,20 @@ async def resolve_event(self, event, minimal=False): event_whitelisted = False event_blacklisted = False for rdtype, children in dns_children.items(): + if event_blacklisted: + break for host in children: if rdtype in ("A", "AAAA", "CNAME"): # having a CNAME to an in-scope resource doesn't make you in-scope - if rdtype != "CNAME": + if not event_whitelisted and rdtype != "CNAME": with suppress(ValidationError): if self.parent_helper.scan.whitelisted(host): - self.log.critical(f"{event_host} --> {host} is whitelisted") event_whitelisted = True # CNAME to a blacklisted resources, means you're blacklisted with suppress(ValidationError): if self.parent_helper.scan.blacklisted(host): - self.log.critical(f"{event_host} --> {host} is blacklisted") event_blacklisted = True + break return event_tags, event_whitelisted, event_blacklisted, dns_children diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 8c6d26eb4..c463391d6 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -919,7 +919,6 @@ def in_tests(self): return os.getenv("BBOT_TESTING", "") == "True" async def _mock_dns(self, mock_data): - self.log.critical(f"SETTING MOCK RESOLVER") from .mock import MockResolver self.resolver = MockResolver(mock_data) From 7bc49b07d3ec59f0eada06ac4d398b91b25e586f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 15:19:41 -0400 Subject: [PATCH 094/171] logging tweaks --- bbot/core/config/logger.py | 17 +++++++++-------- bbot/test/bbot_fixtures.py | 5 ----- bbot/test/test_step_1/test_modules_basic.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 276993ead..9e4d9ed90 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -63,6 +63,8 @@ def __init__(self, core): self.core_logger = logging.getLogger("bbot") self.core = core + self.listener = None + self.process_name = multiprocessing.current_process().name if self.process_name == "MainProcess": self.queue = multiprocessing.Queue() @@ -81,8 +83,7 @@ def setup_queue_handler(self, logging_queue=None): self.queue = logging_queue self.queue_handler = logging.handlers.QueueHandler(logging_queue) - if self.queue_handler not in self.root_logger.handlers: - self.root_logger.addHandler(self.queue_handler) + self.root_logger.addHandler(self.queue_handler) self.core_logger.setLevel(self.log_level) # disable asyncio logging for child processes @@ -153,16 +154,16 @@ def add_log_handler(self, handler, formatter=None): return if handler.formatter is None: handler.setFormatter(debug_format) - for logger in self.loggers: - if handler not in self.listener.handlers: - logger.addHandler(handler) + if handler not in self.listener.handlers: + self.listener.handlers = self.listener.handlers + (handler,) def remove_log_handler(self, handler): if self.listener is None: return - for logger in self.loggers: - if handler in self.listener.handlers: - logger.removeHandler(handler) + if handler in self.listener.handlers: + new_handlers = list(self.listener.handlers) + new_handlers.remove(handler) + self.listener.handlers = tuple(new_handlers) def include_logger(self, logger): if logger not in self.loggers: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 4452cc28f..cf806e14d 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -31,11 +31,6 @@ available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal")) -@pytest.fixture(scope="session", autouse=True) -def setup_logging(): - CORE.logger.setup_queue_handler() - - @pytest.fixture def clean_default_config(monkeypatch): clean_config = OmegaConf.merge( diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index b1891ce05..7f01428e6 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -300,7 +300,7 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_scanner, @pytest.mark.asyncio -async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, monkeypatch, mock_dns): +async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, monkeypatch): from bbot.modules.base import BaseModule class dummy(BaseModule): From 7a7224fb1b81f9efbb579255b42b3bf3be89afb3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 15:37:06 -0400 Subject: [PATCH 095/171] more tests --- bbot/test/test_step_2/module_tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 8db36cd8d..7793530df 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -94,7 +94,7 @@ def set_expect_requests_handler(self, expect_args=None, request_handler=None): async def mock_dns(self, mock_data, scan=None): if scan is None: scan = self.scan - await scan.helpers.dns._dns_mock(mock_data) + await scan.helpers.dns._mock_dns(mock_data) @property def module(self): From 9d2c093f411d72b2b139415eb7f2f4bc6bc0aada Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 20:00:41 -0400 Subject: [PATCH 096/171] reintroduce dns parallelism --- bbot/core/config/logger.py | 4 +- bbot/core/core.py | 5 +- bbot/core/engine.py | 4 +- bbot/core/helpers/dns/dns.py | 4 +- bbot/core/helpers/dns/engine.py | 217 +++++++++--------- bbot/test/test_step_1/test_dns.py | 48 ++-- .../module_tests/test_module_web_report.py | 2 - 7 files changed, 143 insertions(+), 141 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 9e4d9ed90..a5724599d 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -76,7 +76,7 @@ def __init__(self, core): self.log_level = logging.INFO - def setup_queue_handler(self, logging_queue=None): + def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG): if logging_queue is None: logging_queue = self.queue else: @@ -85,7 +85,7 @@ def setup_queue_handler(self, logging_queue=None): self.root_logger.addHandler(self.queue_handler) - self.core_logger.setLevel(self.log_level) + self.core_logger.setLevel(log_level) # disable asyncio logging for child processes if self.process_name != "MainProcess": logging.getLogger("asyncio").setLevel(logging.ERROR) diff --git a/bbot/core/core.py b/bbot/core/core.py index a38307763..1c43e5035 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -26,6 +26,7 @@ class BBOTProcess(multiprocessing.Process): def __init__(self, *args, **kwargs): self.logging_queue = kwargs.pop("logging_queue") + self.log_level = kwargs.pop("log_level") super().__init__(*args, **kwargs) def run(self): @@ -33,7 +34,7 @@ def run(self): try: from bbot.core import CORE - CORE.logger.setup_queue_handler(self.logging_queue) + CORE.logger.setup_queue_handler(self.logging_queue, self.log_level) super().run() except KeyboardInterrupt: log.warning(f"Got KeyboardInterrupt in {self.name}") @@ -165,7 +166,7 @@ def files_config(self): return self._files_config def create_process(self, *args, **kwargs): - process = self.BBOTProcess(*args, logging_queue=self.logger.queue, **kwargs) + process = self.BBOTProcess(*args, logging_queue=self.logger.queue, log_level=self.logger.log_level, **kwargs) process.daemon = True return process diff --git a/bbot/core/engine.py b/bbot/core/engine.py index cf9b90f28..3badd4d42 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -39,7 +39,7 @@ async def run_and_return(self, command, **kwargs): message = self.make_message(command, args=kwargs) await socket.send(message) binary = await socket.recv() - self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") + # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") message = pickle.loads(binary) self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}") # error handling @@ -53,7 +53,7 @@ async def run_and_yield(self, command, **kwargs): await socket.send(message) while 1: binary = await socket.recv() - self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") + # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") message = pickle.loads(binary) self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}") # error handling diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 3bcd92b10..432d0c030 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -82,8 +82,8 @@ async def resolve_batch(self, queries, **kwargs): async for _ in self.run_and_yield("resolve_batch", queries=queries, **kwargs): yield _ - async def resolve_custom_batch(self, queries): - async for _ in self.run_and_yield("resolve_custom_batch", queries=queries): + async def resolve_raw_batch(self, queries): + async for _ in self.run_and_yield("resolve_raw_batch", queries=queries): yield _ async def resolve_event(self, event, minimal=False): diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index c463391d6..7104a732e 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -37,7 +37,7 @@ class DNSEngine(EngineServer): 0: "resolve", 1: "resolve_event", 2: "resolve_batch", - 3: "resolve_custom_batch", + 3: "resolve_raw_batch", 4: "is_wildcard", 5: "is_wildcard_domain", 99: "_mock_dns", @@ -101,7 +101,7 @@ def __init__(self, socket_path, config={}): self.filter_bad_ptrs = self.config.get("dns_filter_ptrs", True) - async def resolve(self, query, include_errors=False, **kwargs): + async def resolve(self, query, **kwargs): """Resolve DNS names and IP addresses to their corresponding results. This is a high-level function that can translate a given domain name to its associated IP addresses @@ -125,23 +125,17 @@ async def resolve(self, query, include_errors=False, **kwargs): results = set() errors = [] try: - r = await self.resolve_raw(query, **kwargs) - if r: - raw_results, errors = r - for rdtype, answers in raw_results: - for answer in answers: - for _, t in self.extract_targets(answer): - results.add(t) + answers, errors = await self.resolve_raw(query, **kwargs) + for answer in answers: + for _, host in self.extract_targets(answer): + results.add(host) except BaseException: log.trace(f"Caught exception in resolve({query}, {kwargs}):") log.trace(traceback.format_exc()) raise self.debug(f"Results for {query} with kwargs={kwargs}: {results}") - if include_errors: - return results, errors - else: - return results + return results async def resolve_raw(self, query, **kwargs): """Resolves the given query to its associated DNS records. @@ -168,39 +162,24 @@ async def resolve_raw(self, query, **kwargs): ([('PTR', )], []) >>> await resolve_raw("dns.google") - ([('A', ), ('AAAA', )], []) + (, []) """ # DNS over TCP is more reliable # But setting this breaks DNS resolution on Ubuntu because systemd-resolve doesn't support TCP # kwargs["tcp"] = True - results = [] - errors = [] try: query = str(query).strip() + kwargs.pop("rdtype", None) + rdtype = kwargs.pop("type", "A") if is_ip(query): - kwargs.pop("type", None) - kwargs.pop("rdtype", None) - results, errors = await self._resolve_ip(query, **kwargs) - return [("PTR", results)], [("PTR", e) for e in errors] + return await self._resolve_ip(query, **kwargs) else: - types = ["A", "AAAA"] - kwargs.pop("rdtype", None) - if "type" in kwargs: - t = kwargs.pop("type") - types = self._parse_rdtype(t, default=types) - for t in types: - r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) - if r: - results.append((t, r)) - for error in e: - errors.append((t, error)) + return await self._resolve_hostname(query, rdtype=rdtype, **kwargs) except BaseException: log.trace(f"Caught exception in resolve_raw({query}, {kwargs}):") log.trace(traceback.format_exc()) raise - return (results, errors) - async def _resolve_hostname(self, query, **kwargs): """Translate a hostname into its corresponding IP addresses. @@ -483,61 +462,57 @@ async def resolve_event(self, event_host, event_type, minimal=False): types = self.all_rdtypes else: types = ("A", "AAAA") - - if types: - for t in types: - resolved_raw, errors = await self.resolve_raw(event_host, type=t, use_cache=True) - for rdtype, e in errors: - if rdtype not in resolved_raw: - event_tags.add(f"{rdtype.lower()}-error") - for rdtype, records in resolved_raw: - rdtype = str(rdtype).upper() - if records: - event_tags.add("resolved") - event_tags.add(f"{rdtype.lower()}-record") - - for r in records: - for _, t in self.extract_targets(r): - if t: - ip = make_ip_type(t) - - if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(t): - self.debug(f"Filtering out bad PTR: {t}") - continue - - try: - dns_children[rdtype].add(ip) - except KeyError: - dns_children[rdtype] = {ip} - - # tag with cloud providers - if not self.in_tests: - to_check = set() - if event_type == "IP_ADDRESS": - to_check.add(event_host) - for rdtype, ips in dns_children.items(): - if rdtype in ("A", "AAAA"): - for ip in ips: - to_check.add(ip) - for ip in to_check: - provider, provider_type, subnet = cloudcheck(ip) - if provider: - event_tags.add(f"{provider_type}-{provider}") - - # if needed, mark as unresolved - if not is_ip(event_host) and "resolved" not in event_tags: - event_tags.add("unresolved") - # check for private IPs + queries = [(event_host, t) for t in types] + async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(queries): + if answers: + rdtype = str(rdtype).upper() + event_tags.add("resolved") + event_tags.add(f"{rdtype.lower()}-record") + + for host, _rdtype in answers: + if host: + host = make_ip_type(host) + + if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(host): + self.debug(f"Filtering out bad PTR: {host}") + continue + + try: + dns_children[_rdtype].add(host) + except KeyError: + dns_children[_rdtype] = {host} + + elif errors: + event_tags.add(f"{rdtype.lower()}-error") + + # tag with cloud providers + if not self.in_tests: + to_check = set() + if event_type == "IP_ADDRESS": + to_check.add(event_host) for rdtype, ips in dns_children.items(): - for ip in ips: - try: - ip = ipaddress.ip_address(ip) - if ip.is_private: - event_tags.add("private-ip") - except ValueError: - continue - - self._event_cache[event_host] = (event_tags, dns_children) + if rdtype in ("A", "AAAA"): + for ip in ips: + to_check.add(ip) + for ip in to_check: + provider, provider_type, subnet = cloudcheck(ip) + if provider: + event_tags.add(f"{provider_type}-{provider}") + + # if needed, mark as unresolved + if not is_ip(event_host) and "resolved" not in event_tags: + event_tags.add("unresolved") + # check for private IPs + for rdtype, ips in dns_children.items(): + for ip in ips: + try: + ip = ipaddress.ip_address(ip) + if ip.is_private: + event_tags.add("private-ip") + except ValueError: + continue + + self._event_cache[event_host] = (event_tags, dns_children) return event_tags, dns_children @@ -594,12 +569,39 @@ async def resolve_batch(self, queries, **kwargs): """ for q in queries: - yield (q, await self.resolve(q, **kwargs)) + results = await self.resolve(q, **kwargs) + # if results: + yield (q, results) - async def resolve_custom_batch(self, queries): - for query, rdtype in queries: - answers, errors = await self.resolve(query, type=rdtype, include_errors=True) - yield ((query, rdtype), (answers, errors)) + async def resolve_raw_batch(self, queries, threads=10): + tasks = {} + + def new_task(query, rdtype): + task = asyncio.create_task(self.resolve_raw(query, type=rdtype)) + tasks[task] = (query, rdtype) + + queries = list(queries) + for _ in range(threads): # Start initial batch of tasks + if queries: # Ensure there are args to process + new_task(*queries.pop(0)) + + while tasks: # While there are tasks pending + # Wait for the first task to complete + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + + for task in done: + answers, errors = task.result() + query, rdtype = tasks.pop(task) + + results = set() + for answer in answers: + for rdtype, host in self.extract_targets(answer): + results.add((host, rdtype)) + # if results or errors: + yield ((query, rdtype), (results, errors)) + + if queries: # Start a new task for each one completed, if URLs remain + new_task(*queries.pop(0)) def extract_targets(self, record): """ @@ -726,9 +728,9 @@ async def is_wildcard(self, query, ips=None, rdtype=None): if ips is None: # then resolve the query for all rdtypes queries = [(query, t) for t in rdtypes_to_check] - async for (query, _rdtype), (answers, errors) in self.resolve_custom_batch(queries): + async for (query, _rdtype), (answers, errors) in self.resolve_raw_batch(queries): if answers: - query_baseline[_rdtype] = answers + query_baseline[_rdtype] = set([a[0] for a in answers]) else: if errors: self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection") @@ -839,22 +841,23 @@ async def is_wildcard_domain(self, domain, log_info=False): # resolve a bunch of random subdomains of the same parent is_wildcard = False wildcard_results = dict() + + queries = [] for rdtype in list(rdtypes_to_check): - # continue if a wildcard was already found for this rdtype - # if rdtype in self._wildcard_cache[host_hash]: - # continue for _ in range(self.wildcard_tests): rand_query = f"{rand_string(digits=False, length=10)}.{host}" - results = await self.resolve(rand_query, type=rdtype, use_cache=False) - if results: - is_wildcard = True - if not rdtype in wildcard_results: - wildcard_results[rdtype] = set() - wildcard_results[rdtype].update(results) - # we know this rdtype is a wildcard - # so we don't need to check it anymore - with suppress(KeyError): - rdtypes_to_check.remove(rdtype) + queries.append((rand_query, rdtype)) + + async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(queries): + if answers: + is_wildcard = True + if not rdtype in wildcard_results: + wildcard_results[rdtype] = set() + wildcard_results[rdtype].update(set(a[0] for a in answers)) + # we know this rdtype is a wildcard + # so we don't need to check it anymore + with suppress(KeyError): + rdtypes_to_check.remove(rdtype) self._wildcard_cache.update({host_hash: wildcard_results}) wildcard_domain_results.update({host: wildcard_results}) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 594192568..1ff9ace92 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -6,7 +6,7 @@ async def test_dns_engine(bbot_scanner): scan = bbot_scanner() result = await scan.helpers.resolve("one.one.one.one") assert "1.1.1.1" in result - assert "2606:4700:4700::1111" in result + assert not "2606:4700:4700::1111" in result results = [_ async for _ in scan.helpers.resolve_batch(("one.one.one.one", "1.1.1.1"))] pass_1 = False @@ -18,13 +18,14 @@ async def test_dns_engine(bbot_scanner): pass_2 = True assert pass_1 and pass_2 - results = [_ async for _ in scan.helpers.resolve_custom_batch((("one.one.one.one", "A"), ("1.1.1.1", "PTR")))] + results = [_ async for _ in scan.helpers.resolve_raw_batch((("one.one.one.one", "A"), ("1.1.1.1", "PTR")))] pass_1 = False pass_2 = False for (query, rdtype), (result, errors) in results: - if query == "one.one.one.one" and "1.1.1.1" in result: + _results = [r[0] for r in result] + if query == "one.one.one.one" and "1.1.1.1" in _results: pass_1 = True - elif query == "1.1.1.1" and "one.one.one.one" in result: + elif query == "1.1.1.1" and "one.one.one.one" in _results: pass_2 = True assert pass_1 and pass_2 @@ -46,23 +47,22 @@ async def test_dns(bbot_scanner): assert ip_responses[0].response.answer[0][0].target.to_text() in ("one.one.one.one.",) # mid level functions - _responses, errors = await dnsengine.resolve_raw("one.one.one.one") + answers, errors = await dnsengine.resolve_raw("one.one.one.one", type="A") responses = [] - for rdtype, response in _responses: - for answers in response: - responses += list(dnsengine.extract_targets(answers)) + for answer in answers: + responses += list(dnsengine.extract_targets(answer)) assert ("A", "1.1.1.1") in responses - _responses, errors = await dnsengine.resolve_raw("one.one.one.one", rdtype="AAAA") + assert not ("AAAA", "2606:4700:4700::1111") in responses + answers, errors = await dnsengine.resolve_raw("one.one.one.one", type="AAAA") responses = [] - for rdtype, response in _responses: - for answers in response: - responses += list(dnsengine.extract_targets(answers)) + for answer in answers: + responses += list(dnsengine.extract_targets(answer)) + assert not ("A", "1.1.1.1") in responses assert ("AAAA", "2606:4700:4700::1111") in responses - _responses, errors = await dnsengine.resolve_raw("1.1.1.1") + answers, errors = await dnsengine.resolve_raw("1.1.1.1") responses = [] - for rdtype, response in _responses: - for answers in response: - responses += list(dnsengine.extract_targets(answers)) + for answer in answers: + responses += list(dnsengine.extract_targets(answer)) assert ("PTR", "one.one.one.one") in responses # high level functions @@ -80,15 +80,11 @@ async def test_dns(bbot_scanner): assert "one.one.one.one" in batch_results["1.1.1.1"] # custom batch resolution - batch_results = [r async for r in dnsengine.resolve_custom_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])] + batch_results = [r async for r in dnsengine.resolve_raw_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])] assert len(batch_results) == 2 batch_results = dict(batch_results) - assert any([x in batch_results[("one.one.one.one", "A")][0] for x in ("1.1.1.1", "1.0.0.1")]) - assert "one.one.one.one" in batch_results[("1.1.1.1", "PTR")][0] - - # "any" type - resolved = await dnsengine.resolve("google.com", type="any") - assert any([scan.helpers.is_subdomain(h) for h in resolved]) + assert ("1.1.1.1", "A") in batch_results[("one.one.one.one", "A")][0] + assert ("one.one.one.one", "PTR") in batch_results[("1.1.1.1", "PTR")][0] # dns cache dnsengine._dns_cache.clear() @@ -103,9 +99,13 @@ async def test_dns(bbot_scanner): await dnsengine.resolve("1.1.1.1") assert hash(f"1.1.1.1:PTR") in dnsengine._dns_cache - await dnsengine.resolve("one.one.one.one") + await dnsengine.resolve("one.one.one.one", type="A") assert hash(f"one.one.one.one:A") in dnsengine._dns_cache + assert not hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache + dnsengine._dns_cache.clear() + await dnsengine.resolve("one.one.one.one", type="AAAA") assert hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache + assert not hash(f"one.one.one.one:A") in dnsengine._dns_cache # Ensure events with hosts have resolved_hosts attribute populated resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", dummy=True) diff --git a/bbot/test/test_step_2/module_tests/test_module_web_report.py b/bbot/test/test_step_2/module_tests/test_module_web_report.py index a37c178e2..c34eef00f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_web_report.py +++ b/bbot/test/test_step_2/module_tests/test_module_web_report.py @@ -13,8 +13,6 @@ async def setup_before_prep(self, module_test): module_test.set_expect_requests(respond_args=respond_args) def check(self, module_test, events): - for e in events: - module_test.log.critical(e) report_file = module_test.scan.home / "web_report.html" with open(report_file) as f: report_content = f.read() From f257fe626c78ba2a64fc2bd30f94a363a3dc34bf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 20:37:56 -0400 Subject: [PATCH 097/171] performance improvements --- bbot/core/helpers/dns/engine.py | 35 +++++++++++++++++++++++++-------- bbot/modules/base.py | 6 ++++-- bbot/scanner/scanner.py | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 7104a732e..c10c2c8a2 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -548,7 +548,7 @@ def event_cache_get(self, host): except KeyError: return set(), set() - async def resolve_batch(self, queries, **kwargs): + async def resolve_batch(self, queries, threads=10, **kwargs): """ A helper to execute a bunch of DNS requests. @@ -566,12 +566,31 @@ async def resolve_batch(self, queries, **kwargs): ... print(result) ('www.evilcorp.com', {'1.1.1.1'}) ('evilcorp.com', {'2.2.2.2'}) - """ - for q in queries: - results = await self.resolve(q, **kwargs) - # if results: - yield (q, results) + tasks = {} + + def new_task(query): + task = asyncio.create_task(self.resolve(query, **kwargs)) + tasks[task] = query + + queries = list(queries) + for _ in range(threads): # Start initial batch of tasks + if queries: # Ensure there are args to process + new_task(queries.pop(0)) + + while tasks: # While there are tasks pending + # Wait for the first task to complete + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + + for task in done: + results = task.result() + query = tasks.pop(task) + + if results: + yield (query, results) + + if queries: # Start a new task for each one completed, if URLs remain + new_task(queries.pop(0)) async def resolve_raw_batch(self, queries, threads=10): tasks = {} @@ -597,8 +616,8 @@ def new_task(query, rdtype): for answer in answers: for rdtype, host in self.extract_targets(answer): results.add((host, rdtype)) - # if results or errors: - yield ((query, rdtype), (results, errors)) + if results or errors: + yield ((query, rdtype), (results, errors)) if queries: # Start a new task for each one completed, if URLs remain new_task(*queries.pop(0)) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 093fa9680..f81f0f81a 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -474,7 +474,7 @@ async def emit_event(self, *args, **kwargs): if event: await self.queue_outgoing_event(event, **emit_kwargs) - async def _events_waiting(self): + async def _events_waiting(self, batch_size=None): """ Asynchronously fetches events from the incoming_event_queue, up to a specified batch size. @@ -492,10 +492,12 @@ async def _events_waiting(self): - "FINISHED" events are handled differently and the finish flag is set to True. - If the queue is empty or the batch size is reached, the loop breaks. """ + if batch_size is None: + batch_size = self.batch_size events = [] finish = False while self.incoming_event_queue: - if len(events) > self.batch_size: + if batch_size != -1 and len(events) > self.batch_size: break try: event = self.incoming_event_queue.get_nowait() diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a849ae6e7..bd17cc089 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -332,7 +332,7 @@ async def async_start(self): break if "python" in self.modules: - events, finish = await self.modules["python"]._events_waiting() + events, finish = await self.modules["python"]._events_waiting(batch_size=-1) for e in events: yield e From 9a29253438b6d5a4659d7c87528a0de7020e3418 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Apr 2024 21:19:24 -0400 Subject: [PATCH 098/171] fix massdns tests --- bbot/modules/massdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index e453ee8d4..a99487ed1 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -233,7 +233,7 @@ async def resolve_and_emit(self): results, source_event, tags = await self.resolve_and_emit_queue.get() self.verbose(f"Resolving batch of {len(results):,} results") async with self._task_counter.count(f"{self.name}.resolve_and_emit()"): - async for hostname, r in self.helpers.resolve_batch(results, type=("A", "CNAME")): + async for hostname, r in self.helpers.resolve_batch(results, type="A"): if not r: self.debug(f"Discarding {hostname} because it didn't resolve") continue From f6f122309534780a7617a38a078c4a6a4e4ecdee Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Apr 2024 10:14:09 -0400 Subject: [PATCH 099/171] remove duplicate preset list --- docs/scanning/presets.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index a2ed4ad2d..f19e27550 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -193,25 +193,3 @@ Conditions use [Jinja](https://palletsprojects.com/p/jinja/), which means they c - `abort(message)` - abort the scan with an optional message If you aren't able to accomplish what you want with conditions, or if you need access to a new variable/function, please let us know on [Github](https://github.com/blacklanternsecurity/bbot/issues/new/choose). - - -## List of Presets - -Here is a full list of BBOT's default presets: - - -| Preset | Category | Description | # Modules | Modules | -|----------------|--------------|------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 52 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | -| code-enum | | Enumerate Git repositories, Docker images, etc. | 6 | dockerhub, github_codesearch, github_org, gitlab, httpx, social | -| dirbust-heavy | web_advanced | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | -| dirbust-light | web_advanced | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | -| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 6 | dehashed, emailformat, hunterio, pgp, skymem, sslcert | -| kitchen-sink | | Everything everywhere all at once | 71 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, gitlab, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, massdns, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wappalyzer, wayback, zoomeye | -| paramminer | web_advanced | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | -| secrets-enum | | | 0 | | -| spider | | Recursive web spider | 1 | httpx | -| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 45 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, threatminer, urlscan, virustotal, wayback, zoomeye | -| web-basic | | Quick web scan | 17 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | -| web-thorough | | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | - From 3fd47c229f679c357f28f57f2c95671971952da6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Apr 2024 14:51:34 -0400 Subject: [PATCH 100/171] restore verbosity toggling --- bbot/cli.py | 61 ++++++++++++++++++++++++++++++++++++++ bbot/core/config/logger.py | 1 + bbot/scanner/scanner.py | 1 - 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/bbot/cli.py b/bbot/cli.py index 4fc84aebf..bcbd0df8d 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -139,6 +139,13 @@ async def _main(): return False if failed else True scan_name = str(scan.name) + + log.verbose("") + log.verbose("### MODULES ###") + log.verbose("") + for row in scan.preset.module_loader.modules_table(scan.preset.scan_modules).splitlines(): + log.verbose(row) + scan.helpers.word_cloud.load() await scan._prep() @@ -150,6 +157,60 @@ async def _main(): log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") input() + import os + import re + import fcntl + from bbot.core.helpers.misc import smart_decode + + def handle_keyboard_input(keyboard_input): + kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") + if keyboard_input: + log.verbose(f'Got keyboard input: "{keyboard_input}"') + kill_match = kill_regex.match(keyboard_input) + if kill_match: + module = kill_match.group("module") + if module in scan.modules: + log.hugewarning(f'Killing module: "{module}"') + scan.manager.kill_module(module, message="killed by user") + else: + log.warning(f'Invalid module: "{module}"') + else: + scan.preset.core.logger.toggle_log_level(logger=log) + scan.manager.modules_status(_log=True) + + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await asyncio.get_event_loop().connect_read_pipe(lambda: protocol, sys.stdin) + + # set stdout and stderr to blocking mode + # this is needed to prevent BlockingIOErrors in logging etc. + fds = [sys.stdout.fileno(), sys.stderr.fileno()] + for fd in fds: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + + async def akeyboard_listen(): + try: + allowed_errors = 10 + while 1: + keyboard_input = None + try: + keyboard_input = smart_decode((await reader.readline()).strip()) + allowed_errors = 10 + except Exception as e: + log_to_stderr(f"Error in keyboard listen loop: {e}", level="TRACE") + log_to_stderr(traceback.format_exc(), level="TRACE") + allowed_errors -= 1 + if keyboard_input is not None: + handle_keyboard_input(keyboard_input) + if allowed_errors <= 0: + break + except Exception as e: + log_to_stderr(f"Error in keyboard listen task: {e}", level="ERROR") + log_to_stderr(traceback.format_exc(), level="TRACE") + + asyncio.create_task(akeyboard_listen()) + await scan.async_start_without_generator() return True diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index a4063027b..39167cfa7 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -226,5 +226,6 @@ def toggle_log_level(self, logger=None): self.set_log_level( self.verbosity_levels_toggle[(i + 1) % len(self.verbosity_levels_toggle)], logger=logger ) + break else: self.set_log_level(self.verbosity_levels_toggle[0], logger=logger) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c859d6c42..8e6f23d87 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -254,7 +254,6 @@ async def _prep(self): await self.load_modules() # run each module's .setup() method - self.info(f"Setting up modules...") succeeded, hard_failed, soft_failed = await self.setup_modules() # abort if there are no output modules From 1108736fad71d587a0804594224204b87d02144e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Apr 2024 15:15:49 -0400 Subject: [PATCH 101/171] ascii art --- bbot/cli.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index bcbd0df8d..4fc75ed43 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,6 +1,19 @@ #!/usr/bin/env python3 import sys +from bbot import __version__ + +ascii_art = f""" ______ _____ ____ _______ + | ___ \| __ \ / __ \__ __| + | |___) | |__) | | | | | | + | ___ <| __ <| | | | | | + | |___) | |__) | |__| | | | + |______/|_____/ \____/ |_| + BIGHUGE BLS OSINT TOOL v{__version__} +""" + +print(ascii_art, file=sys.stderr) + import asyncio import logging import traceback @@ -10,8 +23,6 @@ sys.stdout.reconfigure(line_buffering=True) from bbot.core import CORE - -from bbot import __version__ from bbot.core.errors import * from bbot.core.helpers.logger import log_to_stderr From 39949bd39dec884801a37f9a0ab8307070e54d28 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Apr 2024 16:15:26 -0400 Subject: [PATCH 102/171] move errors.py --- bbot/cli.py | 49 ++++++++++-------- bbot/core/config/files.py | 2 +- bbot/core/errors.py | 74 --------------------------- bbot/core/event/base.py | 2 +- bbot/core/event/helpers.py | 2 +- bbot/core/helpers/diff.py | 2 +- bbot/core/helpers/dns.py | 2 +- bbot/core/helpers/interactsh.py | 2 +- bbot/core/helpers/misc.py | 2 +- bbot/core/helpers/ntlm.py | 2 +- bbot/core/helpers/validators.py | 2 +- bbot/core/helpers/web.py | 2 +- bbot/modules/base.py | 2 +- bbot/modules/bypass403.py | 2 +- bbot/modules/dotnetnuke.py | 2 +- bbot/modules/generic_ssrf.py | 2 +- bbot/modules/host_header.py | 2 +- bbot/modules/ntlm.py | 2 +- bbot/modules/paramminer_headers.py | 2 +- bbot/modules/sslcert.py | 2 +- bbot/modules/url_manipulation.py | 2 +- bbot/scanner/manager.py | 2 +- bbot/scanner/preset/args.py | 2 +- bbot/scanner/preset/conditions.py | 2 +- bbot/scanner/preset/path.py | 2 +- bbot/scanner/preset/preset.py | 2 +- bbot/scanner/scanner.py | 2 +- bbot/scanner/target.py | 2 +- bbot/test/bbot_fixtures.py | 2 +- bbot/test/test_step_1/test_regexes.py | 4 +- 30 files changed, 57 insertions(+), 124 deletions(-) delete mode 100644 bbot/core/errors.py diff --git a/bbot/cli.py b/bbot/cli.py index 4fc75ed43..d12616b92 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,38 +1,41 @@ #!/usr/bin/env python3 import sys +from bbot.errors import * from bbot import __version__ -ascii_art = f""" ______ _____ ____ _______ - | ___ \| __ \ / __ \__ __| - | |___) | |__) | | | | | | - | ___ <| __ <| | | | | | - | |___) | |__) | |__| | | | - |______/|_____/ \____/ |_| - BIGHUGE BLS OSINT TOOL v{__version__} +silent = "-s" in sys.argv or "--silent" in sys.argv + +if not silent: + ascii_art = f"""  ______  _____ ____ _______ + | ___ \| __ \ / __ \__ __| + | |___) | |__) | | | | | | + | ___ <| __ <| | | | | | + | |___) | |__) | |__| | | | + |______/|_____/ \____/ |_| + BIGHUGE BLS OSINT TOOL v{__version__} + + www.blacklanternsecurity.com/bbot """ + print(ascii_art, file=sys.stderr) -print(ascii_art, file=sys.stderr) +scan_name = "" -import asyncio -import logging -import traceback -from contextlib import suppress -# fix tee buffering -sys.stdout.reconfigure(line_buffering=True) +async def _main(): -from bbot.core import CORE -from bbot.core.errors import * -from bbot.core.helpers.logger import log_to_stderr + import asyncio + import logging + import traceback + from contextlib import suppress -log = logging.getLogger("bbot.cli") + # fix tee buffering + sys.stdout.reconfigure(line_buffering=True) -err = False -scan_name = "" + from bbot.core.helpers.logger import log_to_stderr + log = logging.getLogger("bbot.cli") -async def _main(): from bbot.scanner import Scanner from bbot.scanner.preset import Preset @@ -238,6 +241,10 @@ async def akeyboard_listen(): def main(): + import asyncio + import traceback + from bbot.core import CORE + global scan_name try: asyncio.run(_main()) diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 3313b5bab..80704cdbd 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -3,7 +3,7 @@ from omegaconf import OmegaConf from ..helpers.misc import mkdir -from ..errors import ConfigLoadError +from ...errors import ConfigLoadError from ..helpers.logger import log_to_stderr diff --git a/bbot/core/errors.py b/bbot/core/errors.py deleted file mode 100644 index e50e581cd..000000000 --- a/bbot/core/errors.py +++ /dev/null @@ -1,74 +0,0 @@ -class BBOTError(Exception): - pass - - -class ScanError(BBOTError): - pass - - -class ValidationError(BBOTError): - pass - - -class ConfigLoadError(BBOTError): - pass - - -class HttpCompareError(BBOTError): - pass - - -class DirectoryCreationError(BBOTError): - pass - - -class DirectoryDeletionError(BBOTError): - pass - - -class NTLMError(BBOTError): - pass - - -class InteractshError(BBOTError): - pass - - -class WordlistError(BBOTError): - pass - - -class DNSError(BBOTError): - pass - - -class DNSWildcardBreak(DNSError): - pass - - -class CurlError(BBOTError): - pass - - -class PresetNotFoundError(BBOTError): - pass - - -class EnableModuleError(BBOTError): - pass - - -class EnableFlagError(BBOTError): - pass - - -class BBOTArgumentError(BBOTError): - pass - - -class PresetConditionError(BBOTError): - pass - - -class PresetAbortError(PresetConditionError): - pass diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 0d65cca04..4e63b9e53 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -12,7 +12,7 @@ from pydantic import BaseModel, field_validator from .helpers import * -from bbot.core.errors import * +from bbot.errors import * from bbot.core.helpers import ( extract_words, get_file_extension, diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py index d3ad3ee78..b363c9f66 100644 --- a/bbot/core/event/helpers.py +++ b/bbot/core/event/helpers.py @@ -2,7 +2,7 @@ import ipaddress from contextlib import suppress -from bbot.core.errors import ValidationError +from bbot.errors import ValidationError from bbot.core.helpers.regexes import event_type_regexes from bbot.core.helpers import sha1, smart_decode, smart_encode_punycode diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 25b265bde..c21f43718 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -3,7 +3,7 @@ from deepdiff import DeepDiff from contextlib import suppress from xml.parsers.expat import ExpatError -from bbot.core.errors import HttpCompareError +from bbot.errors import HttpCompareError log = logging.getLogger("bbot.core.helpers.diff") diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 1ae8f0ded..3c97a4911 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -13,7 +13,7 @@ from .regexes import dns_name_regex from bbot.core.helpers.ratelimiter import RateLimiter from bbot.core.helpers.async_helpers import NamedLock -from bbot.core.errors import ValidationError, DNSError, DNSWildcardBreak +from bbot.errors import ValidationError, DNSError, DNSWildcardBreak from .misc import is_ip, is_domain, is_dns_name, domain_parents, parent_domain, rand_string, cloudcheck log = logging.getLogger("bbot.core.helpers.dns") diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index aad4a169f..f707fac93 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -11,7 +11,7 @@ from Crypto.PublicKey import RSA from Crypto.Cipher import AES, PKCS1_OAEP -from bbot.core.errors import InteractshError +from bbot.errors import InteractshError log = logging.getLogger("bbot.core.helpers.interactsh") diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index c229b26a1..cc6c086c0 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -14,7 +14,7 @@ from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin # noqa F401 from .url import * # noqa F401 -from .. import errors +from ... import errors from . import regexes as bbot_regexes from .names_generator import random_name, names, adjectives # noqa F401 diff --git a/bbot/core/helpers/ntlm.py b/bbot/core/helpers/ntlm.py index 8605ef34a..9d66b3ea7 100644 --- a/bbot/core/helpers/ntlm.py +++ b/bbot/core/helpers/ntlm.py @@ -5,7 +5,7 @@ import logging import collections -from bbot.core.errors import NTLMError +from bbot.errors import NTLMError log = logging.getLogger("bbot.core.helpers.ntlm") diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py index 0384a876e..c016ed8df 100644 --- a/bbot/core/helpers/validators.py +++ b/bbot/core/helpers/validators.py @@ -5,7 +5,7 @@ from contextlib import suppress from bbot.core.helpers import regexes -from bbot.core.errors import ValidationError +from bbot.errors import ValidationError from bbot.core.helpers.url import parse_url, hash_url from bbot.core.helpers.misc import smart_encode_punycode, split_host_port, make_netloc, is_ip diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 310aaca82..824956c81 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -13,7 +13,7 @@ from httpx._models import Cookies from socksio.exceptions import SOCKSError -from bbot.core.errors import WordlistError, CurlError +from bbot.errors import WordlistError, CurlError from bbot.core.helpers.ratelimiter import RateLimiter from bs4 import MarkupResemblesLocatorWarning diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 093fa9680..f804214a1 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -4,8 +4,8 @@ from sys import exc_info from contextlib import suppress +from ..errors import ValidationError from ..core.helpers.misc import get_size # noqa -from ..core.errors import ValidationError from ..core.helpers.async_helpers import TaskCounter, ShuffleQueue diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py index c58463401..0ce3df899 100644 --- a/bbot/modules/bypass403.py +++ b/bbot/modules/bypass403.py @@ -1,5 +1,5 @@ +from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from bbot.core.errors import HttpCompareError """ Port of https://github.com/iamj0ker/bypass-403/ and https://portswigger.net/bappstore/444407b96d9c4de0adb7aed89e826122 diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py index ea3d7b920..cd3753dc2 100644 --- a/bbot/modules/dotnetnuke.py +++ b/bbot/modules/dotnetnuke.py @@ -1,5 +1,5 @@ +from bbot.errors import InteractshError from bbot.modules.base import BaseModule -from bbot.core.errors import InteractshError class dotnetnuke(BaseModule): diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index 9d75f4a9e..42efa5050 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -1,5 +1,5 @@ +from bbot.errors import InteractshError from bbot.modules.base import BaseModule -from bbot.core.errors import InteractshError ssrf_params = [ diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py index 4adaa766a..3e0f8069f 100644 --- a/bbot/modules/host_header.py +++ b/bbot/modules/host_header.py @@ -1,5 +1,5 @@ +from bbot.errors import InteractshError from bbot.modules.base import BaseModule -from bbot.core.errors import InteractshError class host_header(BaseModule): diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 8f07b4728..93f622566 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -1,4 +1,4 @@ -from bbot.core.errors import NTLMError +from bbot.errors import NTLMError from bbot.modules.base import BaseModule ntlm_discovery_endpoints = [ diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 3458edaa9..0a79532e6 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -1,5 +1,5 @@ +from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from bbot.core.errors import HttpCompareError from bbot.core.helpers.misc import extract_params_json, extract_params_xml, extract_params_html diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index edc4a484f..357826920 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -2,8 +2,8 @@ from OpenSSL import crypto from contextlib import suppress +from bbot.errors import ValidationError from bbot.modules.base import BaseModule -from bbot.core.errors import ValidationError from bbot.core.helpers.async_helpers import NamedLock diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index 983595fe1..74b702eaa 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -1,5 +1,5 @@ +from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from bbot.core.errors import HttpCompareError class url_manipulation(BaseModule): diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 86a2f224a..6193c5e8c 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -3,7 +3,7 @@ import traceback from contextlib import suppress -from ..core.errors import ValidationError +from ..errors import ValidationError from ..core.helpers.async_helpers import TaskCounter, ShuffleQueue log = logging.getLogger("bbot.scanner.manager") diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 943b693cd..1a993e6ab 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -3,7 +3,7 @@ import argparse from omegaconf import OmegaConf -from bbot.core.errors import * +from bbot.errors import * from bbot.core.helpers.misc import chain_lists, get_closest_match log = logging.getLogger("bbot.presets.args") diff --git a/bbot/scanner/preset/conditions.py b/bbot/scanner/preset/conditions.py index 3adacfb91..261a5c76e 100644 --- a/bbot/scanner/preset/conditions.py +++ b/bbot/scanner/preset/conditions.py @@ -1,6 +1,6 @@ import logging -from bbot.core.errors import * +from bbot.errors import * log = logging.getLogger("bbot.preset.conditions") diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index 469c2dd39..ee5235fbf 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from bbot.core.errors import * +from bbot.errors import * log = logging.getLogger("bbot.presets.path") diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 09de8bed7..60dac85dd 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -9,8 +9,8 @@ from .path import PRESET_PATH +from bbot.errors import * from bbot.core import CORE -from bbot.core.errors import * from bbot.core.event.base import make_event from bbot.core.helpers.misc import make_table, mkdir, get_closest_match diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 8e6f23d87..c950b702d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -23,7 +23,7 @@ from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen -from bbot.core.errors import BBOTError, ScanError, ValidationError +from bbot.errors import BBOTError, ScanError, ValidationError log = logging.getLogger("bbot.scanner") diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 77958ff17..38f5d03f5 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -2,7 +2,7 @@ import ipaddress from contextlib import suppress -from bbot.core.errors import * +from bbot.errors import * from bbot.modules.base import BaseModule from bbot.core.event import make_event, is_event from bbot.core.helpers.misc import ip_network_parents, is_ip_type, domain_parents diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index d34cdf730..1960b7ffa 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -12,8 +12,8 @@ from werkzeug.wrappers import Request +from bbot.errors import * # noqa: F401 from bbot.core import CORE -from bbot.core.errors import * # noqa: F401 from bbot.scanner import Preset from bbot.core.helpers.misc import mkdir diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py index 7807e6c79..709674c9e 100644 --- a/bbot/test/test_step_1/test_regexes.py +++ b/bbot/test/test_step_1/test_regexes.py @@ -1,9 +1,9 @@ import pytest import traceback -from bbot.core.event.helpers import get_event_type from bbot.core.helpers import regexes -from bbot.core.errors import ValidationError +from bbot.errors import ValidationError +from bbot.core.event.helpers import get_event_type def test_dns_name_regexes(): From fe618b9c046b619b61e275d0f5cc7d998cb69fc5 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Apr 2024 17:17:31 -0400 Subject: [PATCH 103/171] commit errors.py --- bbot/errors.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 bbot/errors.py diff --git a/bbot/errors.py b/bbot/errors.py new file mode 100644 index 000000000..e50e581cd --- /dev/null +++ b/bbot/errors.py @@ -0,0 +1,74 @@ +class BBOTError(Exception): + pass + + +class ScanError(BBOTError): + pass + + +class ValidationError(BBOTError): + pass + + +class ConfigLoadError(BBOTError): + pass + + +class HttpCompareError(BBOTError): + pass + + +class DirectoryCreationError(BBOTError): + pass + + +class DirectoryDeletionError(BBOTError): + pass + + +class NTLMError(BBOTError): + pass + + +class InteractshError(BBOTError): + pass + + +class WordlistError(BBOTError): + pass + + +class DNSError(BBOTError): + pass + + +class DNSWildcardBreak(DNSError): + pass + + +class CurlError(BBOTError): + pass + + +class PresetNotFoundError(BBOTError): + pass + + +class EnableModuleError(BBOTError): + pass + + +class EnableFlagError(BBOTError): + pass + + +class BBOTArgumentError(BBOTError): + pass + + +class PresetConditionError(BBOTError): + pass + + +class PresetAbortError(PresetConditionError): + pass From 35ecd6c70d4306db5adbe4f670a2069bf53d00f6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 10:33:32 -0400 Subject: [PATCH 104/171] faster event dns caching --- bbot/core/helpers/dns/dns.py | 68 +++++++++++----- bbot/core/helpers/dns/engine.py | 129 +++++++++++++----------------- bbot/test/test_step_1/test_dns.py | 4 + 3 files changed, 108 insertions(+), 93 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 432d0c030..4f25e62fd 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -2,10 +2,12 @@ import logging import dns.exception import dns.asyncresolver +from cachetools import LRUCache from contextlib import suppress from bbot.core.engine import EngineClient from bbot.core.errors import ValidationError +from bbot.core.helpers.async_helpers import NamedLock from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host from .engine import DNSEngine @@ -70,6 +72,10 @@ def __init__(self, parent_helper): self.wildcard_ignore = [] self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) + # event resolution cache + self._event_cache = LRUCache(maxsize=10000) + self._event_cache_locks = NamedLock() + # copy the system's current resolvers to a text file for tool use self.system_resolvers = dns.resolver.Resolver().nameservers # TODO: DNS server speed test (start in background task) @@ -94,29 +100,51 @@ async def resolve_event(self, event, minimal=False): event_host = str(event.host) event_type = str(event.type) - kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} - event_tags, dns_children = await self.run_and_return("resolve_event", **kwargs) - - # whitelisting / blacklisting based on resolved hosts + event_tags = set() + dns_children = dict() event_whitelisted = False event_blacklisted = False - for rdtype, children in dns_children.items(): - if event_blacklisted: - break - for host in children: - if rdtype in ("A", "AAAA", "CNAME"): - # having a CNAME to an in-scope resource doesn't make you in-scope - if not event_whitelisted and rdtype != "CNAME": + + if (not event.host) or (event.type in ("IP_RANGE",)): + return event_tags, event_whitelisted, event_blacklisted, dns_children + + # lock to ensure resolution of the same host doesn't start while we're working here + async with self._event_cache_locks.lock(event_host): + # try to get data from cache + try: + _event_tags, _event_whitelisted, _event_blacklisted, _dns_children = self._event_cache[event_host] + event_tags.update(_event_tags) + # if we found it, return it + if _event_whitelisted is not None: + return event_tags, _event_whitelisted, _event_blacklisted, _dns_children + except KeyError: + pass + + kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} + event_tags, dns_children = await self.run_and_return("resolve_event", **kwargs) + + # whitelisting / blacklisting based on resolved hosts + event_whitelisted = False + event_blacklisted = False + for rdtype, children in dns_children.items(): + if event_blacklisted: + break + for host in children: + if rdtype in ("A", "AAAA", "CNAME"): + # having a CNAME to an in-scope resource doesn't make you in-scope + if not event_whitelisted and rdtype != "CNAME": + with suppress(ValidationError): + if self.parent_helper.scan.whitelisted(host): + event_whitelisted = True + # CNAME to a blacklisted resources, means you're blacklisted with suppress(ValidationError): - if self.parent_helper.scan.whitelisted(host): - event_whitelisted = True - # CNAME to a blacklisted resources, means you're blacklisted - with suppress(ValidationError): - if self.parent_helper.scan.blacklisted(host): - event_blacklisted = True - break - - return event_tags, event_whitelisted, event_blacklisted, dns_children + if self.parent_helper.scan.blacklisted(host): + event_blacklisted = True + break + + self._event_cache[event_host] = (event_tags, event_whitelisted, event_blacklisted, dns_children) + + return event_tags, event_whitelisted, event_blacklisted, dns_children async def is_wildcard(self, query, ips=None, rdtype=None): """ diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index c10c2c8a2..1031cef1f 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -96,8 +96,6 @@ def __init__(self, socket_path, config={}): self._errors = dict() self._debug = self.config.get("dns_debug", False) self._dns_cache = LRUCache(maxsize=10000) - self._event_cache = LRUCache(maxsize=10000) - self._event_cache_locks = NamedLock() self.filter_bad_ptrs = self.config.get("dns_filter_ptrs", True) @@ -440,79 +438,64 @@ async def resolve_event(self, event_host, event_type, minimal=False): dns_children = dict() try: - # lock to ensure resolution of the same host doesn't start while we're working here - async with self._event_cache_locks.lock(event_host): - # try to get data from cache - try: - _event_tags, _dns_children = self._event_cache[event_host] - event_tags.update(_event_tags) - # if we found it, return it - if _event_tags is not None: - return event_tags, _dns_children - except KeyError: - _event_tags, _dns_children = set(), set() - - # then resolve - types = () - if is_ip(event_host): - if not minimal: - types = ("PTR",) + types = () + if is_ip(event_host): + if not minimal: + types = ("PTR",) + else: + if event_type == "DNS_NAME" and not minimal: + types = self.all_rdtypes else: - if event_type == "DNS_NAME" and not minimal: - types = self.all_rdtypes - else: - types = ("A", "AAAA") - queries = [(event_host, t) for t in types] - async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(queries): - if answers: - rdtype = str(rdtype).upper() - event_tags.add("resolved") - event_tags.add(f"{rdtype.lower()}-record") - - for host, _rdtype in answers: - if host: - host = make_ip_type(host) - - if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(host): - self.debug(f"Filtering out bad PTR: {host}") - continue - - try: - dns_children[_rdtype].add(host) - except KeyError: - dns_children[_rdtype] = {host} - - elif errors: - event_tags.add(f"{rdtype.lower()}-error") - - # tag with cloud providers - if not self.in_tests: - to_check = set() - if event_type == "IP_ADDRESS": - to_check.add(event_host) - for rdtype, ips in dns_children.items(): - if rdtype in ("A", "AAAA"): - for ip in ips: - to_check.add(ip) - for ip in to_check: - provider, provider_type, subnet = cloudcheck(ip) - if provider: - event_tags.add(f"{provider_type}-{provider}") - - # if needed, mark as unresolved - if not is_ip(event_host) and "resolved" not in event_tags: - event_tags.add("unresolved") - # check for private IPs + types = ("A", "AAAA") + queries = [(event_host, t) for t in types] + async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(queries): + if answers: + rdtype = str(rdtype).upper() + event_tags.add("resolved") + event_tags.add(f"{rdtype.lower()}-record") + + for host, _rdtype in answers: + if host: + host = make_ip_type(host) + + if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(host): + self.debug(f"Filtering out bad PTR: {host}") + continue + + try: + dns_children[_rdtype].add(host) + except KeyError: + dns_children[_rdtype] = {host} + + elif errors: + event_tags.add(f"{rdtype.lower()}-error") + + # tag with cloud providers + if not self.in_tests: + to_check = set() + if event_type == "IP_ADDRESS": + to_check.add(event_host) for rdtype, ips in dns_children.items(): - for ip in ips: - try: - ip = ipaddress.ip_address(ip) - if ip.is_private: - event_tags.add("private-ip") - except ValueError: - continue - - self._event_cache[event_host] = (event_tags, dns_children) + if rdtype in ("A", "AAAA"): + for ip in ips: + to_check.add(ip) + for ip in to_check: + provider, provider_type, subnet = cloudcheck(ip) + if provider: + event_tags.add(f"{provider_type}-{provider}") + + # if needed, mark as unresolved + if not is_ip(event_host) and "resolved" not in event_tags: + event_tags.add("unresolved") + # check for private IPs + for rdtype, ips in dns_children.items(): + for ip in ips: + try: + ip = ipaddress.ip_address(ip) + if ip.is_private: + event_tags.add("private-ip") + except ValueError: + continue return event_tags, dns_children diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 1ff9ace92..07beca1f2 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -110,9 +110,13 @@ async def test_dns(bbot_scanner): # Ensure events with hosts have resolved_hosts attribute populated resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", dummy=True) resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", dummy=True) + assert resolved_hosts_event1.host not in scan.helpers.dns._event_cache + assert resolved_hosts_event2.host not in scan.helpers.dns._event_cache event_tags1, event_whitelisted1, event_blacklisted1, children1 = await scan.helpers.resolve_event( resolved_hosts_event1 ) + assert resolved_hosts_event1.host in scan.helpers.dns._event_cache + assert resolved_hosts_event2.host in scan.helpers.dns._event_cache event_tags2, event_whitelisted2, event_blacklisted2, children2 = await scan.helpers.resolve_event( resolved_hosts_event2 ) From 19329cbf42d097b87ffcac7d65f8d57bc8e1389d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 10:56:58 -0400 Subject: [PATCH 105/171] remove obsolete massdns logic --- bbot/cli.py | 3 +- bbot/core/config/files.py | 2 +- bbot/core/config/logger.py | 2 +- bbot/core/helpers/dns/dns.py | 2 +- bbot/core/helpers/dns/engine.py | 2 +- bbot/core/helpers/logger.py | 52 ---------------------- bbot/core/modules.py | 2 +- bbot/modules/massdns.py | 76 ++++++++++----------------------- bbot/modules/output/human.py | 2 +- 9 files changed, 30 insertions(+), 113 deletions(-) delete mode 100644 bbot/core/helpers/logger.py diff --git a/bbot/cli.py b/bbot/cli.py index d12616b92..b3ac3f430 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -3,6 +3,7 @@ import sys from bbot.errors import * from bbot import __version__ +from bbot.logger import log_to_stderr silent = "-s" in sys.argv or "--silent" in sys.argv @@ -32,8 +33,6 @@ async def _main(): # fix tee buffering sys.stdout.reconfigure(line_buffering=True) - from bbot.core.helpers.logger import log_to_stderr - log = logging.getLogger("bbot.cli") from bbot.scanner import Scanner diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 80704cdbd..1df185c15 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -4,7 +4,7 @@ from ..helpers.misc import mkdir from ...errors import ConfigLoadError -from ..helpers.logger import log_to_stderr +from ...logger import log_to_stderr bbot_code_dir = Path(__file__).parent.parent.parent diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 40ea3c96f..b6aec39aa 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -7,7 +7,7 @@ from pathlib import Path from ..helpers.misc import mkdir, error_and_exit -from ..helpers.logger import colorize, loglevel_mapping +from ...logger import colorize, loglevel_mapping debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s") diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 4f25e62fd..58e00a9b1 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -5,8 +5,8 @@ from cachetools import LRUCache from contextlib import suppress +from bbot.errors import ValidationError from bbot.core.engine import EngineClient -from bbot.core.errors import ValidationError from bbot.core.helpers.async_helpers import NamedLock from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 1031cef1f..2b5903292 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -9,8 +9,8 @@ from cachetools import LRUCache from ..regexes import dns_name_regex +from bbot.errors import DNSWildcardBreak from bbot.core.engine import EngineServer -from bbot.core.errors import DNSWildcardBreak from bbot.core.helpers.async_helpers import NamedLock from bbot.core.helpers.misc import ( clean_dns_record, diff --git a/bbot/core/helpers/logger.py b/bbot/core/helpers/logger.py deleted file mode 100644 index b70d4b4b4..000000000 --- a/bbot/core/helpers/logger.py +++ /dev/null @@ -1,52 +0,0 @@ -import sys - -loglevel_mapping = { - "DEBUG": "DBUG", - "TRACE": "TRCE", - "VERBOSE": "VERB", - "HUGEVERBOSE": "VERB", - "INFO": "INFO", - "HUGEINFO": "INFO", - "SUCCESS": "SUCC", - "HUGESUCCESS": "SUCC", - "WARNING": "WARN", - "HUGEWARNING": "WARN", - "ERROR": "ERRR", - "CRITICAL": "CRIT", -} -color_mapping = { - "DEBUG": 242, # grey - "TRACE": 242, # red - "VERBOSE": 242, # grey - "INFO": 69, # blue - "HUGEINFO": 69, # blue - "SUCCESS": 118, # green - "HUGESUCCESS": 118, # green - "WARNING": 208, # orange - "HUGEWARNING": 208, # orange - "ERROR": 196, # red - "CRITICAL": 196, # red -} -color_prefix = "\033[1;38;5;" -color_suffix = "\033[0m" - - -def colorize(s, level="INFO"): - seq = color_mapping.get(level, 15) # default white - colored = f"{color_prefix}{seq}m{s}{color_suffix}" - return colored - - -def log_to_stderr(msg, level="INFO", logname=True): - """ - Print to stderr with BBOT logger colors - """ - levelname = level.upper() - if not any(x in sys.argv for x in ("-s", "--silent")): - levelshort = f"[{loglevel_mapping.get(level, 'INFO')}]" - levelshort = f"{colorize(levelshort, level=levelname)}" - if levelname == "CRITICAL" or levelname.startswith("HUGE"): - msg = colorize(msg, level=levelname) - if logname: - msg = f"{levelshort} {msg}" - print(msg, file=sys.stderr) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 62929230c..2f3ce445c 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -13,9 +13,9 @@ from contextlib import suppress from bbot.core import CORE +from bbot.logger import log_to_stderr from .flags import flag_descriptions -from .helpers.logger import log_to_stderr from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform, mkdir diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index a99487ed1..cad536dfd 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -1,7 +1,6 @@ import re import json import random -import asyncio import subprocess from bbot.modules.templates.subdomain_enum import subdomain_enum @@ -101,10 +100,8 @@ async def setup(self): cache_hrs=24 * 7, ) self.devops_mutations = list(self.helpers.word_cloud.devops_mutations) - self.mutation_run = 1 + self._mutation_run = 1 - self.resolve_and_emit_queue = asyncio.Queue() - self.resolve_and_emit_task = asyncio.create_task(self.resolve_and_emit()) return await super().setup() async def filter_event(self, event): @@ -137,8 +134,8 @@ async def handle_event(self, event): query = self.make_query(event) self.source_events.add_target(event) self.info(f"Brute-forcing subdomains for {query} (source: {event.data})") - results = await self.massdns(query, self.subdomain_list) - await self.resolve_and_emit_queue.put((results, event, None)) + for hostname in await self.massdns(query, self.subdomain_list): + await self.emit_result(hostname, event, query) def abort_if(self, event): if not event.scope_distance == 0: @@ -149,6 +146,13 @@ def abort_if(self, event): return True, "event is unresolved" return False, "" + async def emit_result(self, result, source_event, query, tags=None): + if not result == source_event: + kwargs = {"abort_if": self.abort_if} + if tags is not None: + kwargs["tags"] = tags + await self.emit_event(result, "DNS_NAME", source_event, **kwargs) + def already_processed(self, hostname): if hash(hostname) in self.processed: return True @@ -221,35 +225,6 @@ async def massdns(self, domain, subdomains): # everything checks out return results - async def resolve_and_emit(self): - """ - When results are found, they are placed into self.resolve_and_emit_queue. - The purpose of this function (which is started as a task in the module's setup()) is to consume results from - the queue, resolve them, and if they resolve, emit them. - - This exists to prevent disrupting the scan with huge batches of DNS resolutions. - """ - while 1: - results, source_event, tags = await self.resolve_and_emit_queue.get() - self.verbose(f"Resolving batch of {len(results):,} results") - async with self._task_counter.count(f"{self.name}.resolve_and_emit()"): - async for hostname, r in self.helpers.resolve_batch(results, type="A"): - if not r: - self.debug(f"Discarding {hostname} because it didn't resolve") - continue - self.add_found(hostname) - if source_event is None: - source_event = self.source_events.get(hostname) - if source_event is None: - self.warning(f"Could not correlate source event from: {hostname}") - source_event = self.scan.root_event - kwargs = {"abort_if": self.abort_if, "tags": tags} - await self.emit_event(hostname, "DNS_NAME", source_event, **kwargs) - - @property - def running(self): - return super().running or self.resolve_and_emit_queue.qsize() > 0 - async def _canary_check(self, domain, num_checks=50): random_subdomains = list(self.gen_random_subdomains(num_checks)) self.verbose(f"Testing {len(random_subdomains):,} canaries against {domain}") @@ -378,9 +353,6 @@ def add_mutation(_domain_hash, m): self.mutations_tried.add(h) mutations.add(m) - num_base_mutations = len(base_mutations) - self.debug(f"Base mutations for {domain}: {num_base_mutations:,}") - # try every subdomain everywhere else for _domain, _subdomains in found: if _domain == domain: @@ -388,7 +360,10 @@ def add_mutation(_domain_hash, m): for s in _subdomains: first_segment = s.split(".")[0] # skip stuff with lots of numbers (e.g. PTRs) - if self.has_excessive_digits(first_segment): + digits = self.digit_regex.findall(first_segment) + excessive_digits = len(digits) > 2 + long_digits = any(len(d) > 3 for d in digits) + if excessive_digits or long_digits: continue add_mutation(domain_hash, first_segment) for word in self.helpers.extract_words( @@ -396,9 +371,6 @@ def add_mutation(_domain_hash, m): ): add_mutation(domain_hash, word) - num_massdns_mutations = len(mutations) - num_base_mutations - self.debug(f"Mutations from previous subdomains for {domain}: {num_massdns_mutations:,}") - # numbers + devops mutations for mutation in self.helpers.word_cloud.mutations( subdomains, cloud=False, numbers=3, number_padding=1 @@ -407,26 +379,24 @@ def add_mutation(_domain_hash, m): m = delimiter.join(mutation).lower() add_mutation(domain_hash, m) - num_word_cloud_mutations = len(mutations) - num_massdns_mutations - self.debug(f"Mutations added by word cloud for {domain}: {num_word_cloud_mutations:,}") - # special dns mutator - self.debug( - f"DNS Mutator size: {len(self.helpers.word_cloud.dns_mutator):,} (limited to {self.max_mutations:,})" - ) for subdomain in self.helpers.word_cloud.dns_mutator.mutations( subdomains, max_mutations=self.max_mutations ): add_mutation(domain_hash, subdomain) - num_mutations = len(mutations) - num_word_cloud_mutations - self.debug(f"Mutations added by DNS Mutator: {num_mutations:,}") - if mutations: self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})") results = list(await self.massdns(query, mutations)) + for hostname in results: + source_event = self.source_events.get(hostname) + if source_event is None: + self.warning(f"Could not correlate source event from: {hostname}") + source_event = self.scan.root_event + await self.emit_result( + hostname, source_event, query, tags=[f"mutation-{self._mutation_run}"] + ) if results: - await self.resolve_and_emit_queue.put((results, None, [f"mutation-{self.mutation_run}"])) found_mutations = True continue break @@ -434,7 +404,7 @@ def add_mutation(_domain_hash, m): self.warning(e) if found_mutations: - self.mutation_run += 1 + self._mutation_run += 1 def add_found(self, host): if not isinstance(host, str): diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py index e1f4746c4..389a4bd84 100644 --- a/bbot/modules/output/human.py +++ b/bbot/modules/output/human.py @@ -1,6 +1,6 @@ from contextlib import suppress -from bbot.core.helpers.logger import log_to_stderr +from bbot.logger import log_to_stderr from bbot.modules.output.base import BaseOutputModule From b54b567cb8c89e297b97da23036d09b43a403d32 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 10:57:09 -0400 Subject: [PATCH 106/171] add logger.py --- bbot/logger.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 bbot/logger.py diff --git a/bbot/logger.py b/bbot/logger.py new file mode 100644 index 000000000..b70d4b4b4 --- /dev/null +++ b/bbot/logger.py @@ -0,0 +1,52 @@ +import sys + +loglevel_mapping = { + "DEBUG": "DBUG", + "TRACE": "TRCE", + "VERBOSE": "VERB", + "HUGEVERBOSE": "VERB", + "INFO": "INFO", + "HUGEINFO": "INFO", + "SUCCESS": "SUCC", + "HUGESUCCESS": "SUCC", + "WARNING": "WARN", + "HUGEWARNING": "WARN", + "ERROR": "ERRR", + "CRITICAL": "CRIT", +} +color_mapping = { + "DEBUG": 242, # grey + "TRACE": 242, # red + "VERBOSE": 242, # grey + "INFO": 69, # blue + "HUGEINFO": 69, # blue + "SUCCESS": 118, # green + "HUGESUCCESS": 118, # green + "WARNING": 208, # orange + "HUGEWARNING": 208, # orange + "ERROR": 196, # red + "CRITICAL": 196, # red +} +color_prefix = "\033[1;38;5;" +color_suffix = "\033[0m" + + +def colorize(s, level="INFO"): + seq = color_mapping.get(level, 15) # default white + colored = f"{color_prefix}{seq}m{s}{color_suffix}" + return colored + + +def log_to_stderr(msg, level="INFO", logname=True): + """ + Print to stderr with BBOT logger colors + """ + levelname = level.upper() + if not any(x in sys.argv for x in ("-s", "--silent")): + levelshort = f"[{loglevel_mapping.get(level, 'INFO')}]" + levelshort = f"{colorize(levelshort, level=levelname)}" + if levelname == "CRITICAL" or levelname.startswith("HUGE"): + msg = colorize(msg, level=levelname) + if logname: + msg = f"{levelshort} {msg}" + print(msg, file=sys.stderr) From 8c1ba8884f00ea2bd03ab54030c1e8aebc411c58 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 11:02:13 -0400 Subject: [PATCH 107/171] remove vscode artifacts --- .vscode/launch.json | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 2c3f13080..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "0.1.0", - "configurations": [ - { - "name": "bbot", - "type": "python", - "request": "launch", - "cwd": "${workspaceFolder}", - "module": "bbot.cli", - "args": ["-m httpx", "-t example.com"] - } - ] -} \ No newline at end of file From 465e68d94681033ae6ff097cdc89fd07eab0357e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 14:17:18 -0400 Subject: [PATCH 108/171] remove redundant v in version --- bbot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/cli.py b/bbot/cli.py index d12616b92..76c94b4b5 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -13,7 +13,7 @@ | ___ <| __ <| | | | | | | |___) | |__) | |__| | | | |______/|_____/ \____/ |_| - BIGHUGE BLS OSINT TOOL v{__version__} + BIGHUGE BLS OSINT TOOL {__version__} www.blacklanternsecurity.com/bbot """ From 16fe38a9b64be025b039c3985ea28246c036cbb0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 14:17:54 -0400 Subject: [PATCH 109/171] version begins with v --- bbot/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/__init__.py b/bbot/__init__.py index 7c3310ef7..8e016095f 100644 --- a/bbot/__init__.py +++ b/bbot/__init__.py @@ -1,2 +1,2 @@ # version placeholder (replaced by poetry-dynamic-versioning) -__version__ = "0.0.0" +__version__ = "v0.0.0" From 74aa6faa700b78de7e87edb7da6b25d074f904d0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 16:55:59 -0400 Subject: [PATCH 110/171] fix output module slowness --- bbot/modules/base.py | 6 ++++-- bbot/presets/spider.yml | 14 -------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index f804214a1..6c4a4a4a1 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -474,7 +474,7 @@ async def emit_event(self, *args, **kwargs): if event: await self.queue_outgoing_event(event, **emit_kwargs) - async def _events_waiting(self): + async def _events_waiting(self, batch_size=None): """ Asynchronously fetches events from the incoming_event_queue, up to a specified batch size. @@ -492,10 +492,12 @@ async def _events_waiting(self): - "FINISHED" events are handled differently and the finish flag is set to True. - If the queue is empty or the batch size is reached, the loop breaks. """ + if batch_size is None: + batch_size = self.batch_size events = [] finish = False while self.incoming_event_queue: - if len(events) > self.batch_size: + if batch_size != -1 and len(events) > self.batch_size: break try: event = self.incoming_event_queue.get_nowait() diff --git a/bbot/presets/spider.yml b/bbot/presets/spider.yml index 3a1ef5de4..fa0635e06 100644 --- a/bbot/presets/spider.yml +++ b/bbot/presets/spider.yml @@ -10,17 +10,3 @@ config: web_spider_depth: 4 # maximum number of links to follow per page web_spider_links_per_page: 25 - - modules: - stdout: - format: text - # only output URLs to the console - event_types: - - URL - # only show in-scope URLs - in_scope_only: True - # display the raw URLs, nothing else - event_fields: - - data - # automatically dedupe - accept_dups: False From 4c354022cf5355b46bc28d95504f121515d9a90c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Apr 2024 17:00:35 -0400 Subject: [PATCH 111/171] again --- bbot/scanner/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c950b702d..4b94b7388 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -331,7 +331,7 @@ async def async_start(self): break if "python" in self.modules: - events, finish = await self.modules["python"]._events_waiting() + events, finish = await self.modules["python"]._events_waiting(-1) for e in events: yield e From fdd4c6e826e92a1101d1af51df7eb71888501e37 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 5 Apr 2024 11:26:21 -0400 Subject: [PATCH 112/171] remove custom stdout settings until better solution --- bbot/presets/email-enum.yml | 15 --------------- bbot/presets/subdomain-enum.yml | 15 --------------- 2 files changed, 30 deletions(-) diff --git a/bbot/presets/email-enum.yml b/bbot/presets/email-enum.yml index 9f391e28b..a5ffd6e3c 100644 --- a/bbot/presets/email-enum.yml +++ b/bbot/presets/email-enum.yml @@ -5,18 +5,3 @@ flags: output_modules: - emails - -config: - modules: - stdout: - format: text - # only output EMAIL_ADDRESSes to the console - event_types: - - EMAIL_ADDRESS - # only show in-scope emails - in_scope_only: True - # display the raw emails, nothing else - event_fields: - - data - # automatically dedupe - accept_dups: False diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index 1741a06e5..f18789ee6 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -5,18 +5,3 @@ flags: output_modules: - subdomains - -config: - modules: - stdout: - format: text - # only output DNS_NAMEs to the console - event_types: - - DNS_NAME - # only show in-scope subdomains - in_scope_only: True - # display the raw subdomains, nothing else - event_fields: - - data - # automatically dedupe - accept_dups: False From 8223aa38e39748348915b59e3c9d7c03ba2685a1 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 5 Apr 2024 16:44:33 -0400 Subject: [PATCH 113/171] better cli error handling --- bbot/cli.py | 9 ++++----- bbot/core/config/files.py | 2 +- bbot/core/config/logger.py | 2 +- bbot/core/modules.py | 2 +- bbot/{core/helpers => }/logger.py | 0 bbot/modules/output/human.py | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) rename bbot/{core/helpers => }/logger.py (100%) diff --git a/bbot/cli.py b/bbot/cli.py index 76c94b4b5..910b078d6 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import sys +import logging from bbot.errors import * from bbot import __version__ +from bbot.logger import log_to_stderr silent = "-s" in sys.argv or "--silent" in sys.argv @@ -25,15 +27,12 @@ async def _main(): import asyncio - import logging import traceback from contextlib import suppress # fix tee buffering sys.stdout.reconfigure(line_buffering=True) - from bbot.core.helpers.logger import log_to_stderr - log = logging.getLogger("bbot.cli") from bbot.scanner import Scanner @@ -155,9 +154,9 @@ async def _main(): scan_name = str(scan.name) log.verbose("") - log.verbose("### MODULES ###") + log.verbose("### MODULES ENABLED ###") log.verbose("") - for row in scan.preset.module_loader.modules_table(scan.preset.scan_modules).splitlines(): + for row in scan.preset.module_loader.modules_table(scan.preset.modules).splitlines(): log.verbose(row) scan.helpers.word_cloud.load() diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 80704cdbd..6547d02ec 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -3,8 +3,8 @@ from omegaconf import OmegaConf from ..helpers.misc import mkdir +from ...logger import log_to_stderr from ...errors import ConfigLoadError -from ..helpers.logger import log_to_stderr bbot_code_dir = Path(__file__).parent.parent.parent diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 39167cfa7..ca4b30593 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -5,7 +5,7 @@ from pathlib import Path from ..helpers.misc import mkdir, error_and_exit -from ..helpers.logger import colorize, loglevel_mapping +from ...logger import colorize, loglevel_mapping debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s") diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 62929230c..507f189c8 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -15,7 +15,7 @@ from bbot.core import CORE from .flags import flag_descriptions -from .helpers.logger import log_to_stderr +from bbot.logger import log_to_stderr from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform, mkdir diff --git a/bbot/core/helpers/logger.py b/bbot/logger.py similarity index 100% rename from bbot/core/helpers/logger.py rename to bbot/logger.py diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py index e1f4746c4..389a4bd84 100644 --- a/bbot/modules/output/human.py +++ b/bbot/modules/output/human.py @@ -1,6 +1,6 @@ from contextlib import suppress -from bbot.core.helpers.logger import log_to_stderr +from bbot.logger import log_to_stderr from bbot.modules.output.base import BaseOutputModule From 04bc2060cd10979f501f24e57d187b3402c38688 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 6 Apr 2024 21:35:36 -0400 Subject: [PATCH 114/171] prune unnecessary asyncio tasks --- bbot/modules/base.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 6c4a4a4a1..1536107ae 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -394,8 +394,7 @@ async def _handle_batch(self): self.verbose(f"Handling batch of {len(events):,} events") submitted = True async with self.scan._acatch(f"{self.name}.handle_batch()"): - handle_batch_task = asyncio.create_task(self.handle_batch(*events)) - await handle_batch_task + await self.handle_batch(*events) self.verbose(f"Finished handling batch of {len(events):,} events") if finish: context = f"{self.name}.finish()" @@ -554,8 +553,7 @@ async def _setup(self): status = False self.debug(f"Setting up module {self.name}") try: - setup_task = asyncio.create_task(self.setup()) - result = await setup_task + result = await self.setup() if type(result) == tuple and len(result) == 2: status, msg = result else: @@ -622,16 +620,13 @@ async def _worker(self): if event.type == "FINISHED": context = f"{self.name}.finish()" async with self.scan._acatch(context), self._task_counter.count(context): - finish_task = asyncio.create_task(self.finish()) - await finish_task + await self.finish() else: context = f"{self.name}.handle_event({event})" self.scan.stats.event_consumed(event, self) self.debug(f"Handling {event}") async with self.scan._acatch(context), self._task_counter.count(context): - task_name = f"{self.name}.handle_event({event})" - handle_event_task = asyncio.create_task(self.handle_event(event), name=task_name) - await handle_event_task + await self.handle_event(event) self.debug(f"Finished handling {event}") else: self.debug(f"Not accepting {event} because {reason}") From f899bb9368a75a201b6de23fced8d17d128679a6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 7 Apr 2024 04:35:41 -0400 Subject: [PATCH 115/171] remove unnecessary checks in deduplication tests --- .../test_step_1/test_manager_deduplication.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 435e57991..7ff5ea4e3 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -133,23 +133,23 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 5 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"]) assert len(default_events) == 6 - assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) - assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert len(all_events) == 26 - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"]) @@ -164,31 +164,31 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 5 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"]) assert len(no_suppress_dupes) == 6 - assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) - assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert len(accept_dupes) == 10 - assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert len(per_hostport_only) == 6 - assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) + assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) - assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"]) - assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only" and e.source.data == "test.notreal"]) + assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) + assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data]) assert len(per_domain_only) == 1 From 60049ac40fdb88806652f17d617c70543fc52628 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 7 Apr 2024 04:55:04 -0400 Subject: [PATCH 116/171] make sure zmq sockets are always cleaned up at the end of a scan --- bbot/core/engine.py | 7 +++++-- bbot/scanner/scanner.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 3badd4d42..06f965c45 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -33,6 +33,7 @@ def __init__(self, **kwargs): self.server_kwargs = kwargs.pop("server_kwargs", {}) self.server_process = self.start_server() self.context = zmq.asyncio.Context() + atexit.register(self.cleanup) async def run_and_return(self, command, **kwargs): with self.new_socket() as socket: @@ -111,6 +112,10 @@ def new_socket(self): finally: socket.close() + def cleanup(self): + # delete socket file on exit + self.socket_path.unlink(missing_ok=True) + class EngineServer: @@ -126,8 +131,6 @@ def __init__(self, socket_path): self.socket = self.context.socket(zmq.ROUTER) # create socket file self.socket.bind(f"ipc://{socket_path}") - # delete socket file on exit - atexit.register(socket_path.unlink, missing_ok=True) async def run_and_return(self, client_id, command_fn, **kwargs): self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({kwargs})") diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 6ce949cb2..c7aaf8f44 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -644,8 +644,12 @@ async def _cleanup(self): None """ self.status = "CLEANING_UP" + # clean up dns engine + self.helpers.dns.cleanup() + # clean up modules for mod in self.modules.values(): await mod._cleanup() + # clean up self if not self._cleanedup: self._cleanedup = True with contextlib.suppress(Exception): From 6998f5951928c3f46c12b2350227741e5d71b232 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 Apr 2024 10:04:14 -0400 Subject: [PATCH 117/171] commit discord bot example --- docs/dev/discord_bot.md | 75 ++--------------------------------------- examples/discord_bot.py | 72 +++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 75 insertions(+), 73 deletions(-) create mode 100644 examples/discord_bot.py diff --git a/docs/dev/discord_bot.md b/docs/dev/discord_bot.md index 9c67efb0a..ff2aa860a 100644 --- a/docs/dev/discord_bot.md +++ b/docs/dev/discord_bot.md @@ -3,77 +3,6 @@ Below is a simple Discord bot designed to run BBOT scans. -```python -import asyncio -import discord -from discord.ext import commands - -from bbot.scanner import Scanner -from bbot.modules import MODULE_LOADER -from bbot.modules.output.discord import Discord - - -class BBOTDiscordBot(commands.Cog): - """ - A simple Discord bot capable of running a BBOT scan. - - To set up: - 1. Go to Discord Developer Portal (https://discord.com/developers) - 2. Create a new application - 3. Create an invite link for the bot, visit the link to invite it to your server - - Your Application --> OAuth2 --> URL Generator - - For Scopes, select "bot"" - - For Bot Permissions, select: - - Read Messages/View Channels - - Send Messages - 4. Turn on "Message Content Intent" - - Your Application --> Bot --> Privileged Gateway Intents --> Message Content Intent - 5. Copy your Discord Bot Token and put it at the top this file - - Your Application --> Bot --> Reset Token - 6. Run this script - - To scan evilcorp.com, you would type: - - /scan evilcorp.com - - Results will be output to the same channel. - """ - def __init__(self): - self.current_scan = None - - @commands.command(name="scan", description="Scan a target with BBOT.") - async def scan(self, ctx, target: str): - if self.current_scan is not None: - self.current_scan.stop() - await ctx.send(f"Starting scan against {target}.") - - # creates scan instance - self.current_scan = Scanner(target, flags="subdomain-enum") - discord_module = Discord(self.current_scan) - - seen = set() - num_events = 0 - # start scan and iterate through results - async for event in self.current_scan.async_start(): - if hash(event) in seen: - continue - seen.add(hash(event)) - await ctx.send(discord_module.format_message(event)) - num_events += 1 - - await ctx.send(f"Finished scan against {target}. {num_events:,} results.") - self.current_scan = None - - -if __name__ == "__main__": - intents = discord.Intents.default() - intents.message_content = True - bot = commands.Bot(command_prefix="/", intents=intents) - - @bot.event - async def on_ready(): - print(f"We have logged in as {bot.user}") - await bot.add_cog(BBOTDiscordBot()) - - bot.run("DISCORD_BOT_TOKEN_HERE") +```python title="examples/discord_bot.py" +--8<-- "examples/discord_bot.py" ``` diff --git a/examples/discord_bot.py b/examples/discord_bot.py new file mode 100644 index 000000000..61e8bc2b0 --- /dev/null +++ b/examples/discord_bot.py @@ -0,0 +1,72 @@ +import asyncio +import discord +from discord.ext import commands + +from bbot.scanner import Scanner +from bbot.modules.output.discord import Discord + + +class BBOTDiscordBot(commands.Cog): + """ + A simple Discord bot capable of running a BBOT scan. + + To set up: + 1. Go to Discord Developer Portal (https://discord.com/developers) + 2. Create a new application + 3. Create an invite link for the bot, visit the link to invite it to your server + - Your Application --> OAuth2 --> URL Generator + - For Scopes, select "bot"" + - For Bot Permissions, select: + - Read Messages/View Channels + - Send Messages + 4. Turn on "Message Content Intent" + - Your Application --> Bot --> Privileged Gateway Intents --> Message Content Intent + 5. Copy your Discord Bot Token and put it at the top this file + - Your Application --> Bot --> Reset Token + 6. Run this script + + To scan evilcorp.com, you would type: + + /scan evilcorp.com + + Results will be output to the same channel. + """ + + def __init__(self): + self.current_scan = None + + @commands.command(name="scan", description="Scan a target with BBOT.") + async def scan(self, ctx, target: str): + if self.current_scan is not None: + self.current_scan.stop() + await ctx.send(f"Starting scan against {target}.") + + # creates scan instance + self.current_scan = Scanner(target, flags="subdomain-enum") + discord_module = Discord(self.current_scan) + + seen = set() + num_events = 0 + # start scan and iterate through results + async for event in self.current_scan.async_start(): + if hash(event) in seen: + continue + seen.add(hash(event)) + await ctx.send(discord_module.format_message(event)) + num_events += 1 + + await ctx.send(f"Finished scan against {target}. {num_events:,} results.") + self.current_scan = None + + +if __name__ == "__main__": + intents = discord.Intents.default() + intents.message_content = True + bot = commands.Bot(command_prefix="/", intents=intents) + + @bot.event + async def on_ready(): + print(f"We have logged in as {bot.user}") + await bot.add_cog(BBOTDiscordBot()) + + bot.run("DISCORD_BOT_TOKEN_HERE") diff --git a/mkdocs.yml b/mkdocs.yml index 6cb486614..af7fab579 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -98,6 +98,7 @@ markdown_extensions: - attr_list - admonition - pymdownx.details + - pymdownx.snippets - pymdownx.superfences - pymdownx.highlight: use_pygments: True From b2121a200b50e71e97a401f683c4d95bfe246bef Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 Apr 2024 15:11:43 -0400 Subject: [PATCH 118/171] WIP module hooks --- README.md | 2 +- bbot/core/event/base.py | 4 +- bbot/core/helpers/dns/dns.py | 44 ++---- bbot/modules/base.py | 93 +++++++++++- bbot/modules/httpx.py | 2 - bbot/modules/internal/dnsresolve.py | 75 ++++++++++ bbot/scanner/manager.py | 142 ++++++++----------- bbot/scanner/scanner.py | 38 +++-- poetry.lock | 213 +++++++++++++++++++++++++++- pyproject.toml | 1 + 10 files changed, 474 insertions(+), 140 deletions(-) create mode 100644 bbot/modules/internal/dnsresolve.py diff --git a/README.md b/README.md index 1a6f79d9d..1ac582cae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![bbot_banner](https://user-images.githubusercontent.com/20261699/158000235-6c1ace81-a267-4f8e-90a1-f4c16884ebac.png)](https://github.com/blacklanternsecurity/bbot) -#### BBOT /ˈBEE·bot/ (noun): A recursive internet scanner for hackers. +#### /ˈBEE·bot/ (noun): A recursive internet scanner for hackers. [![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index cd6533dd4..bf030ebc2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -154,6 +154,7 @@ def __init__( self._priority = None self._module_priority = None self._resolved_hosts = set() + self.dns_children = dict() # keep track of whether this event has been recorded by the scan self._stats_recorded = False @@ -210,9 +211,6 @@ def __init__( if _internal: # or source._internal: self.internal = True - # an event indicating whether the event has undergone DNS resolution - self._resolved = asyncio.Event() - # inherit web spider distance from parent self.web_spider_distance = getattr(self.source, "web_spider_distance", 0) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 58e00a9b1..f9c505540 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -102,8 +102,6 @@ async def resolve_event(self, event, minimal=False): event_type = str(event.type) event_tags = set() dns_children = dict() - event_whitelisted = False - event_blacklisted = False if (not event.host) or (event.type in ("IP_RANGE",)): return event_tags, event_whitelisted, event_blacklisted, dns_children @@ -112,39 +110,20 @@ async def resolve_event(self, event, minimal=False): async with self._event_cache_locks.lock(event_host): # try to get data from cache try: - _event_tags, _event_whitelisted, _event_blacklisted, _dns_children = self._event_cache[event_host] + _event_tags, _dns_children = self._event_cache[event_host] event_tags.update(_event_tags) # if we found it, return it if _event_whitelisted is not None: - return event_tags, _event_whitelisted, _event_blacklisted, _dns_children + return event_tags, _dns_children except KeyError: pass kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} event_tags, dns_children = await self.run_and_return("resolve_event", **kwargs) - # whitelisting / blacklisting based on resolved hosts - event_whitelisted = False - event_blacklisted = False - for rdtype, children in dns_children.items(): - if event_blacklisted: - break - for host in children: - if rdtype in ("A", "AAAA", "CNAME"): - # having a CNAME to an in-scope resource doesn't make you in-scope - if not event_whitelisted and rdtype != "CNAME": - with suppress(ValidationError): - if self.parent_helper.scan.whitelisted(host): - event_whitelisted = True - # CNAME to a blacklisted resources, means you're blacklisted - with suppress(ValidationError): - if self.parent_helper.scan.blacklisted(host): - event_blacklisted = True - break - - self._event_cache[event_host] = (event_tags, event_whitelisted, event_blacklisted, dns_children) + self._event_cache[event_host] = (event_tags, dns_children) - return event_tags, event_whitelisted, event_blacklisted, dns_children + return event_tags, dns_children async def is_wildcard(self, query, ips=None, rdtype=None): """ @@ -204,7 +183,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): async def is_wildcard_domain(self, domain, log_info=False): return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) - async def handle_wildcard_event(self, event, children): + async def handle_wildcard_event(self, event): """ Used within BBOT's scan manager to detect and tag DNS wildcard events. @@ -212,19 +191,18 @@ async def handle_wildcard_event(self, event, children): is overwritten, for example: `_wildcard.evilcorp.com`. Args: - event (object): The event to check for wildcards. - children (list): A list of the event's resulting DNS children after resolution. + event (Event): The event to check for wildcards. Returns: None: This method modifies the `event` in place and does not return a value. Examples: - >>> handle_wildcard_event(event, children) + >>> handle_wildcard_event(event) # The `event` might now have tags like ["wildcard", "a-wildcard", "aaaa-wildcard"] and # its `data` attribute might be modified to "_wildcard.evilcorp.com" if it was detected # as a wildcard. """ - log.debug(f"Entering handle_wildcard_event({event}, children={children})") + log.debug(f"Entering handle_wildcard_event({event}, children={event.dns_children})") try: event_host = str(event.host) # wildcard checks @@ -239,10 +217,10 @@ async def handle_wildcard_event(self, event, children): event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if not is_ip(event.host) and children: + if (not is_ip(event.host)) and event.dns_children: if wildcard_rdtypes: # these are the rdtypes that successfully resolve - resolved_rdtypes = set([c.upper() for c in children]) + resolved_rdtypes = set([c.upper() for c in event.dns_children]) # these are the rdtypes that have wildcards wildcard_rdtypes_set = set(wildcard_rdtypes) # consider the event a full wildcard if all its records are wildcards @@ -276,7 +254,7 @@ async def handle_wildcard_event(self, event, children): # event.add_tag(f"{rdtype.lower()}-wildcard-domain") finally: - log.debug(f"Finished handle_wildcard_event({event}, children={children})") + log.debug(f"Finished handle_wildcard_event({event}, children={event.dns_children})") async def _mock_dns(self, mock_data): from .mock import MockResolver diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 1536107ae..a50203759 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -111,6 +111,7 @@ class BaseModule: _priority = 3 _name = "base" _type = "scan" + _hook = False def __init__(self, scan): """Initializes a module instance. @@ -590,7 +591,7 @@ async def _worker(self): - Each event is subject to a post-check via '_event_postcheck()' to decide whether it should be handled. - Special 'FINISHED' events trigger the 'finish()' method of the module. """ - async with self.scan._acatch(context=self._worker): + async with self.scan._acatch(context=self._worker, unhandled_is_critical=True): try: while not self.scan.stopping and not self.errored: # hold the reigns if our outgoing queue is full @@ -691,7 +692,10 @@ async def _event_postcheck(self, event): """ A simple wrapper for dup tracking """ - acceptable, reason = await self.__event_postcheck(event) + # special exception for "FINISHED" event + if event.type in ("FINISHED",): + return True, "" + acceptable, reason = await self._event_postcheck_inner(event) if acceptable: # check duplicates is_incoming_duplicate, reason = self.is_incoming_duplicate(event, add=True) @@ -700,7 +704,7 @@ async def _event_postcheck(self, event): return acceptable, reason - async def __event_postcheck(self, event): + async def _event_postcheck_inner(self, event): """ Post-checks an event to determine if it should be accepted by the module for handling. @@ -718,10 +722,6 @@ async def __event_postcheck(self, event): - This method also maintains host-based tracking when the `per_host_only` or similar flags are set. - The method will also update event production stats for output modules. """ - # special exception for "FINISHED" event - if event.type in ("FINISHED",): - return True, "" - # force-output certain events to the graph if self._is_graph_important(event): return True, "event is critical to the graph" @@ -1399,3 +1399,82 @@ def critical(self, *args, trace=True, **kwargs): self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs) if trace: self.trace() + + +class HookModule(BaseModule): + accept_dupes = True + suppress_dupes = False + _hook = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._first = False + + async def _worker(self): + async with self.scan._acatch(context=self._worker, unhandled_is_critical=True): + try: + while not self.scan.stopping and not self.errored: + + try: + if self.incoming_event_queue is not False: + incoming = await self.get_incoming_event() + try: + event, _kwargs = incoming + except ValueError: + event = incoming + _kwargs = {} + else: + self.debug(f"Event queue is in bad state") + break + except asyncio.queues.QueueEmpty: + await asyncio.sleep(0.1) + continue + + if event.type == "FINISHED": + context = f"{self.name}.finish()" + async with self.scan._acatch(context), self._task_counter.count(context): + finish_task = asyncio.create_task(self.finish()) + await finish_task + continue + + self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") + async with self._task_counter.count(f"event_postcheck({event})"): + acceptable, reason = await self._event_postcheck(event) + + if acceptable: + context = f"{self.name}.handle_event({event})" + self.scan.stats.event_consumed(event, self) + self.debug(f"Hooking {event}") + async with self.scan._acatch(context), self._task_counter.count(context): + task_name = f"{self.name}.handle_event({event})" + handle_event_task = asyncio.create_task(self.handle_event(event), name=task_name) + await handle_event_task + self.debug(f"Finished hooking {event}") + else: + self.debug(f"Not hooking {event} because {reason}") + + await self.outgoing_event_queue.put((event, _kwargs)) + + except asyncio.CancelledError: + self.log.trace("Worker cancelled") + raise + self.log.trace(f"Worker stopped") + + async def get_incoming_event(self): + try: + return self.incoming_event_queue.get_nowait() + except asyncio.queues.QueueEmpty: + if self._first: + return self.scan.manager.get_event_from_modules() + raise + + async def queue_event(self, event, precheck=False): + try: + self.incoming_event_queue.put_nowait(event) + if event.type != "FINISHED": + self.scan.manager._new_activity = True + except AttributeError: + self.debug(f"Not in an acceptable state to queue incoming event") + + async def _event_postcheck(self, event): + return await self._event_postcheck_inner(event) diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 30cc3fba3..0f74fbcfc 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -173,8 +173,6 @@ async def handle_batch(self, *events): if url_event: if url_event != source_event: await self.emit_event(url_event) - else: - url_event._resolved.set() # HTTP response await self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags) diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py new file mode 100644 index 000000000..62ba4362a --- /dev/null +++ b/bbot/modules/internal/dnsresolve.py @@ -0,0 +1,75 @@ +from bbot.modules.base import HookModule + + +class dnsresolve(HookModule): + hooked_events = ["DNS_NAME"] + _priority = 1 + + async def setup(self): + self.dns_resolution = self.scan.config.get("dns_resolution", False) + return True + + async def handle_event(self, event): + event.add_tag("dnsresolved") + resolved_hosts = set() + dns_children = {} + dns_tags = set() + + # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist + # this is a performance optimization and it'd be nice if we could do it for all events not just targets + # but for non-target events, we need to know what IPs they resolve to so we can make scope decisions about them + skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist + if skip_dns_resolution: + dns_tags = {"resolved"} + else: + # DNS resolution + dns_tags, dns_children = await self.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) + + # whitelisting / blacklisting based on resolved hosts + event_whitelisted = False + event_blacklisted = False + for rdtype, children in dns_children.items(): + if event_blacklisted: + break + for host in children: + if rdtype in ("A", "AAAA", "CNAME"): + for ip in ips: + resolved_hosts.add(ip) + # having a CNAME to an in-scope resource doesn't make you in-scope + if not event_whitelisted and rdtype != "CNAME": + with suppress(ValidationError): + if self.parent_helper.scan.whitelisted(host): + event_whitelisted = True + # CNAME to a blacklisted resources, means you're blacklisted + with suppress(ValidationError): + if self.parent_helper.scan.blacklisted(host): + event_blacklisted = True + break + + # kill runaway DNS chains + dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) + if dns_resolve_distance >= self.helpers.dns.max_dns_resolve_distance: + log.debug( + f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.scan.helpers.dns.max_dns_resolve_distance})" + ) + dns_children = {} + + if event.type in ("DNS_NAME", "IP_ADDRESS"): + event._dns_children = dns_children + for tag in dns_tags: + event.add_tag(tag) + + event._resolved_hosts = resolved_hosts + + event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event) + event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event) + if event_blacklisted: + event.add_tag("blacklisted") + reason = "event host" + if event_blacklisted_dns: + reason = "DNS associations" + log.debug(f"Omitting due to blacklisted {reason}: {event}") + return + + if event_whitelisted: + event.add_tag("whitelisted") diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 6193c5e8c..4d6a0e239 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -49,9 +49,39 @@ def __init__(self, scan): self._task_counter = TaskCounter() self._new_activity = True self._modules_by_priority = None + self._hook_modules = None + self._non_hook_modules = None self._incoming_queues = None self._module_priority_weights = None + async def _worker_loop(self): + try: + while not self.scan.stopped: + try: + async with self._task_counter.count("get_event_from_modules()"): + # if we have hooks set up, we always get events from the last (lowest priority) hook module. + if self.hook_modules: + last_hook_module = self.hook_modules[-1] + incoming = last_hook_module.outgoing_event_queue.get_nowait() + else: + # otherwise, we go through all the modules + incoming = self.get_event_from_modules() + try: + event, kwargs = incoming + except: + log.critical(incoming) + except asyncio.queues.QueueEmpty: + await asyncio.sleep(0.1) + continue + async with self._task_counter.count(f"emit_event({event})"): + emit_event_task = asyncio.create_task( + self.emit_event(event, **kwargs), name=f"emit_event({event})" + ) + await emit_event_task + + except Exception: + log.critical(traceback.format_exc()) + async def init_events(self): """ Initializes events by seeding the scanner with target events and distributing them for further processing. @@ -63,9 +93,9 @@ async def init_events(self): context = f"manager.init_events()" async with self.scan._acatch(context), self._task_counter.count(context): - await self.distribute_event(self.scan.root_event) + sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) - for event in sorted_events: + for event in [self.scan.root_event] + sorted_events: event._dummy = False event.scope_distance = 0 event.web_spider_distance = 0 @@ -75,7 +105,11 @@ async def init_events(self): if event.module is None: event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") self.scan.verbose(f"Target: {event}") - self.queue_event(event) + if self.hook_modules: + first_hook_module = self.hook_modules[0] + await first_hook_module.queue_event(event) + else: + self.queue_event(event) await asyncio.sleep(0.1) self.scan._finished_init = True @@ -97,21 +131,17 @@ async def emit_event(self, event, *args, **kwargs): # skip event if it fails precheck if event.type != "DNS_NAME": acceptable = self._event_precheck(event) - if not acceptable: - event._resolved.set() - return log.debug(f'Module "{event.module}" raised {event}') if quick: log.debug(f"Quick-emitting {event}") - event._resolved.set() for kwarg in callbacks: kwargs.pop(kwarg, None) async with self.scan._acatch(context=self.distribute_event): await self.distribute_event(event) else: - async with self.scan._acatch(context=self._emit_event, finally_callback=event._resolved.set): + async with self.scan._acatch(context=self._emit_event): await self._emit_event( event, *args, @@ -174,53 +204,7 @@ async def _emit_event(self, event, **kwargs): on_success_callback = kwargs.pop("on_success_callback", None) abort_if = kwargs.pop("abort_if", None) - # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist - skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist - if skip_dns_resolution: - event._resolved.set() - dns_children = {} - dns_tags = {"resolved"} - event_whitelisted_dns = True - event_blacklisted_dns = False - resolved_hosts = [] - else: - # DNS resolution - ( - dns_tags, - event_whitelisted_dns, - event_blacklisted_dns, - dns_children, - ) = await self.scan.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) - resolved_hosts = set() - for rdtype, ips in dns_children.items(): - if rdtype in ("A", "AAAA", "CNAME"): - for ip in ips: - resolved_hosts.add(ip) - - # kill runaway DNS chains - dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) - if dns_resolve_distance >= self.scan.helpers.dns.max_dns_resolve_distance: - log.debug( - f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.scan.helpers.dns.max_dns_resolve_distance})" - ) - dns_children = {} - - if event.type in ("DNS_NAME", "IP_ADDRESS"): - event._dns_children = dns_children - for tag in dns_tags: - event.add_tag(tag) - - event._resolved_hosts = resolved_hosts - - event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event) - event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event) - if event_blacklisted: - event.add_tag("blacklisted") - reason = "event host" - if event_blacklisted_dns: - reason = "DNS associations" - log.debug(f"Omitting due to blacklisted {reason}: {event}") - return + event_whitelisted = "whitelisted" in event.tags # other blacklist rejections - URL extensions, etc. if "blacklisted" in event.tags: @@ -244,7 +228,7 @@ async def _emit_event(self, event, **kwargs): if event.scope_distance <= self.scan.scope_search_distance: if not "unresolved" in event.tags: if not self.scan.helpers.is_ip_type(event.host): - await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) + await self.scan.helpers.dns.handle_wildcard_event(event) # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() if event.type == "DNS_NAME": @@ -301,8 +285,8 @@ async def _emit_event(self, event, **kwargs): if emit_children: dns_child_events = [] - if dns_children: - for rdtype, records in dns_children.items(): + if event.dns_children: + for rdtype, records in event.dns_children.items(): module = self.scan._make_dummy_module_dns(rdtype) module._priority = 4 for record in records: @@ -329,7 +313,6 @@ async def _emit_event(self, event, **kwargs): log.trace(traceback.format_exc()) finally: - event._resolved.set() log.debug(f"{event.module}.emit_event() finished for {event}") def is_incoming_duplicate(self, event, add=False): @@ -395,30 +378,15 @@ async def distribute_event(self, event): if not is_outgoing_duplicate and -1 < event.scope_distance < 1: self.scan.word_cloud.absorb_event(event) for mod in self.scan.modules.values(): + # don't distribute events to hook modules + if mod._hook: + continue acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes # graph_important = mod._type == "output" and event._graph_important == True graph_important = mod._is_graph_important(event) if acceptable_dup or graph_important: await mod.queue_event(event) - async def _worker_loop(self): - try: - while not self.scan.stopped: - try: - async with self._task_counter.count("get_event_from_modules()"): - event, kwargs = self.get_event_from_modules() - except asyncio.queues.QueueEmpty: - await asyncio.sleep(0.1) - continue - async with self._task_counter.count(f"emit_event({event})"): - emit_event_task = asyncio.create_task( - self.emit_event(event, **kwargs), name=f"emit_event({event})" - ) - await emit_event_task - - except Exception: - log.critical(traceback.format_exc()) - def kill_module(self, module_name, message=None): from signal import SIGINT @@ -438,7 +406,7 @@ def modules_by_priority(self): @property def incoming_queues(self): if not self._incoming_queues: - queues_by_priority = [m.outgoing_event_queue for m in self.modules_by_priority] + queues_by_priority = [m.outgoing_event_queue for m in self.modules_by_priority if not m._hook] self._incoming_queues = [self.incoming_event_queue] + queues_by_priority return self._incoming_queues @@ -453,10 +421,24 @@ def incoming_qsize(self): def module_priority_weights(self): if not self._module_priority_weights: # we subtract from six because lower priorities == higher weights - priorities = [5] + [6 - m.priority for m in self.modules_by_priority] + priorities = [5] + [6 - m.priority for m in self.modules_by_priority if not m._hook] self._module_priority_weights = priorities return self._module_priority_weights + @property + def hook_modules(self): + if self._hook_modules is None: + self._hook_modules = [m for m in self.modules_by_priority if m._hook] + if self._hook_modules: + self._hook_modules[0]._first = True + return self._hook_modules + + @property + def non_hook_modules(self): + if self._non_hook_modules is None: + self._non_hook_modules = [m for m in self.modules_by_priority if not m._hook] + return self._non_hook_modules + def get_event_from_modules(self): for q in self.scan.helpers.weighted_shuffle(self.incoming_queues, self.module_priority_weights): try: @@ -485,8 +467,6 @@ def queue_event(self, event, **kwargs): event_in_scope = self.scan.whitelisted(event) and not self.scan.blacklisted(event) if not event_in_scope: event.module_priority += event.scope_distance - # Wait for parent event to resolve (in case its scope distance changes) - # await resolved = event.source._resolved.wait() # update event's scope distance based on its parent event.scope_distance = event.source.scope_distance + 1 self.incoming_event_queue.put_nowait((event, kwargs)) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c7aaf8f44..e2b813cbc 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -316,13 +316,13 @@ async def async_start(self): asyncio.create_task(self.manager._worker_loop()) for _ in range(self.max_workers) ] - # distribute seed events - self.init_events_task = asyncio.create_task(self.manager.init_events()) - self.status = "RUNNING" self._start_modules() self.verbose(f"{len(self.modules):,} modules started") + # distribute seed events + self.init_events_task = asyncio.create_task(self.manager.init_events()) + # main scan loop while 1: # abort if we're aborting @@ -330,12 +330,13 @@ async def async_start(self): self._drain_queues() break + # yield events as they come (async for event in scan.async_start()) if "python" in self.modules: events, finish = await self.modules["python"]._events_waiting(batch_size=-1) for e in events: yield e - # if initialization finished and the scan is no longer active + # break if initialization finished and the scan is no longer active if self._finished_init and not self.manager.active: new_activity = await self.finish() if not new_activity: @@ -387,7 +388,17 @@ async def async_start(self): def _start_modules(self): self.verbose(f"Starting module worker loops") - for module_name, module in self.modules.items(): + + # hook modules get sewn together like human centipede + if len(self.manager.hook_modules) > 1: + for i, hook_module in enumerate(self.manager.hook_modules[:-1]): + next_hook_module = self.manager.hook_modules[i + 1] + self.debug( + f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" + ) + hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue + + for module in self.modules.values(): module.start() async def setup_modules(self, remove_failed=True): @@ -552,8 +563,8 @@ async def finish(self): self.status = "FINISHING" # Trigger .finished() on every module and start over log.info("Finishing scan") - finished_event = self.make_event("FINISHED", "FINISHED", dummy=True) for module in self.modules.values(): + finished_event = self.make_event(f"FINISHED", "FINISHED", dummy=True, tags={module.name}) await module.queue_event(finished_event) self.verbose("Completed finish()") return True @@ -767,7 +778,6 @@ def root_event(self): root_event = self.make_event(data=f"{self.name} ({self.id})", event_type="SCAN", dummy=True) root_event._id = self.id root_event.scope_distance = 0 - root_event._resolved.set() root_event.source = root_event root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") return root_event @@ -993,7 +1003,7 @@ async def _status_ticker(self, interval=15): self.manager.modules_status(_log=True) @contextlib.asynccontextmanager - async def _acatch(self, context="scan", finally_callback=None): + async def _acatch(self, context="scan", finally_callback=None, unhandled_is_critical=False): """ Async version of catch() @@ -1003,9 +1013,9 @@ async def _acatch(self, context="scan", finally_callback=None): try: yield except BaseException as e: - self._handle_exception(e, context=context) + self._handle_exception(e, context=context, unhandled_is_critical=unhandled_is_critical) - def _handle_exception(self, e, context="scan", finally_callback=None): + def _handle_exception(self, e, context="scan", finally_callback=None, unhandled_is_critical=False): if callable(context): context = f"{context.__qualname__}()" filename, lineno, funcname = self.helpers.get_traceback_details(e) @@ -1018,8 +1028,12 @@ def _handle_exception(self, e, context="scan", finally_callback=None): elif isinstance(e, asyncio.CancelledError): raise elif isinstance(e, Exception): - log.error(f"Error in {context}: {filename}:{lineno}:{funcname}(): {e}") - log.trace(traceback.format_exc()) + if unhandled_is_critical: + log.critical(f"Error in {context}: {filename}:{lineno}:{funcname}(): {e}") + log.critical(traceback.format_exc()) + else: + log.error(f"Error in {context}: {filename}:{lineno}:{funcname}(): {e}") + log.trace(traceback.format_exc()) if callable(finally_callback): finally_callback(e) diff --git a/poetry.lock b/poetry.lock index 94601cf20..7be6743c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -695,6 +695,32 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "h2" +version = "4.1.0" +description = "HTTP/2 State-Machine based protocol implementation" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "h2-4.1.0-py3-none-any.whl", hash = "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d"}, + {file = "h2-4.1.0.tar.gz", hash = "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"}, +] + +[package.dependencies] +hpack = ">=4.0,<5" +hyperframe = ">=6.0,<7" + +[[package]] +name = "hpack" +version = "4.0.0" +description = "Pure-Python HPACK header compression" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "hpack-4.0.0-py3-none-any.whl", hash = "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c"}, + {file = "hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"}, +] + [[package]] name = "httpcore" version = "1.0.5" @@ -740,6 +766,17 @@ cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +[[package]] +name = "hyperframe" +version = "6.0.1" +description = "HTTP/2 framing layer for Python" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "hyperframe-6.0.1-py3-none-any.whl", hash = "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15"}, + {file = "hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"}, +] + [[package]] name = "identify" version = "2.5.35" @@ -827,6 +864,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "kiss-headers" +version = "2.4.3" +description = "Object-oriented HTTP and IMAP (structured) headers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "kiss_headers-2.4.3-py3-none-any.whl", hash = "sha256:9d800b77532068e8748be9f96f30eaeb547cdc5345e4689ddf07b77071256239"}, + {file = "kiss_headers-2.4.3.tar.gz", hash = "sha256:70c689ce167ac83146f094ea916b40a3767d67c2e05a4cb95b0fd2e33bf243f1"}, +] + [[package]] name = "libsass" version = "0.23.0" @@ -1304,6 +1352,30 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "niquests" +version = "3.5.5" +description = "Niquests is a simple, yet elegant, HTTP library. It is a drop-in replacement for Requests, which is under feature freeze." +optional = false +python-versions = ">=3.7" +files = [ + {file = "niquests-3.5.5-py3-none-any.whl", hash = "sha256:bd134c7cbc414661840e73bebe0b766c16321558b3c444efb3f63aad9189e308"}, + {file = "niquests-3.5.5.tar.gz", hash = "sha256:5b52183cd4ee16f360de1e5b97bc266b933e8603320102d10d17f68a95e926ba"}, +] + +[package.dependencies] +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +kiss-headers = ">=2,<4" +urllib3-future = ">=2.7.900,<3" +wassima = ">=1.0.1,<2" + +[package.extras] +http3 = ["urllib3-future[qh3]"] +ocsp = ["cryptography (>=41.0.0,<43.0.0)"] +socks = ["urllib3-future[socks]"] +speedups = ["orjson (>=3,<4)", "urllib3-future[brotli,zstd]"] + [[package]] name = "nodeenv" version = "1.8.0" @@ -2083,6 +2155,47 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "qh3" +version = "0.15.1" +description = "An implementation of QUIC and HTTP/3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "qh3-0.15.1-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fe8f15e9fe5850508188ce38bdc89bda03d1a99ce3c2fbde6ee02d1d91edc557"}, + {file = "qh3-0.15.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:114d04dd51d3d9eca76ce804fea60ccb0fcbe84be08dcca70f32e30e5736aa00"}, + {file = "qh3-0.15.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:265240539a630cf458f3651f08bd07e4d46b2bf941a25e7f594321401701b30d"}, + {file = "qh3-0.15.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1074ee0e30fe825b60bd113767b56dcfe2f155e79f893d5180d4fd2adebaa1de"}, + {file = "qh3-0.15.1-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0afd9e7b90c90ff3e8c8e376020e3753936da0ce8db57ebb9fc95a50ba7e015d"}, + {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b7c4df89b03f90f67e372693c70f357dabc18908cb07dab21aa550c4f777017b"}, + {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:3923bb17dbdf91f060cb3b04cb8c2e3bf74d528a26f4c0e5365e311bade33b58"}, + {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:87b61b59e3c692b70384430ccf634a228c54bb38ee6d974d76a7b086b356ecad"}, + {file = "qh3-0.15.1-cp37-abi3-win32.whl", hash = "sha256:3d02314850b0c8a5cd39015b9f5e5b21d54980702e3e80dcfc6aa7b983d7494a"}, + {file = "qh3-0.15.1-cp37-abi3-win_amd64.whl", hash = "sha256:1a0305b389cec13af879dee32c6584cff45a52865456e6645d84023ed8442d67"}, + {file = "qh3-0.15.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8c000a32d2d3dadf252a55d71f676011f02c0e529024176d35e53122293d8a54"}, + {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9562c2648a0e468cc3c97e77c658c0b9db288e29cfc79d52220e50ddcfac9fe9"}, + {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71ab5d62606556c0ba2b1f3bf118dcb2d6f0236add792ffba42845a741abe498"}, + {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:592bd246325090ffe8324761808713b1c99c7b7cae37ec4bd2841d0054729422"}, + {file = "qh3-0.15.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f68ac19161aaef887351f2e8df1972d91726ade69105b4ae1653ab0e70a18536"}, + {file = "qh3-0.15.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a842e65e57f8092f1fa185b1dc95556b1b695f06a4eb48dc9c07f018bd7a7ec"}, + {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bca27698ad110fabda026f844f453b1ac1a1e2d86729846f5be0cdc9e7df419"}, + {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c87a0613efbc3d353a76a917044270caf43198890ffe702b3cbe9b44065c45e"}, + {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77673a9b02e19c4f81e419efa2aa4040dec10f0a6158788196d8b5ef6aafb0d9"}, + {file = "qh3-0.15.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:087da39ebd5a8608e8df0892860b4fdcd4ff83753d7312cead490de6f1bce504"}, + {file = "qh3-0.15.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:561ba4d84e617ecc0d7506f532da2814e672a06cdcb903209616f00c5da74c14"}, + {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec733c6a4da5ecf4448434562aba617ecfabbdef0a58df812684db7d03000070"}, + {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74cc03a94e605820f3c5882e47388e8d2d8616d51db57a6e5120d9f2344dc04a"}, + {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:939253ceaf5664c4e90f6317f0097839b6c8af627bb5905181f4fcbbc209c395"}, + {file = "qh3-0.15.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:969582d286be3e468ff5e53cdf2a5f47a942ea370f870a0276c4235a7ed13a71"}, + {file = "qh3-0.15.1.tar.gz", hash = "sha256:816c787f68855a28aa703be54956b21ff258e1650978a06b98a23bbf252cbe7e"}, +] + +[package.dependencies] +cryptography = ">=41.0.0,<43" + +[package.extras] +dev = ["coverage[toml] (>=7.2.2)"] + [[package]] name = "regex" version = "2023.12.25" @@ -2401,6 +2514,28 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "urllib3-future" +version = "2.7.903" +description = "urllib3.future is a powerful HTTP 1.1, 2, and 3 client with both sync and async interfaces" +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3_future-2.7.903-py3-none-any.whl", hash = "sha256:04bebce1291c9be9db2b03bb016db56d1f7e3dbe425c7250129552a8ceaf6827"}, + {file = "urllib3_future-2.7.903.tar.gz", hash = "sha256:99e1265c8bb2478d86b8a6c0de991ac275ad58d5e43ac11d980a0dd1cc183804"}, +] + +[package.dependencies] +h11 = ">=0.11.0,<1.0.0" +h2 = ">=4.0.0,<5.0.0" +qh3 = {version = ">=0.14.0,<1.0.0", markers = "(platform_system == \"Darwin\" or platform_system == \"Windows\" or platform_system == \"Linux\") and (platform_python_implementation == \"CPython\" or (platform_python_implementation == \"PyPy\" and python_version >= \"3.8\" and python_version < \"3.11\"))"} + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +qh3 = ["qh3 (>=0.14.0,<1.0.0)"] +socks = ["python-socks (>=2.0,<3.0)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "virtualenv" version = "20.25.1" @@ -2421,6 +2556,82 @@ platformdirs = ">=3.9.1,<5" docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +[[package]] +name = "wassima" +version = "1.1.0" +description = "Access your OS root certificates with the atmost ease" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wassima-1.1.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6b67781f7b9483a5dbcb1cabe588ab316f06f7c97a9d60b6981681f790aa16a1"}, + {file = "wassima-1.1.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fb331ab3ff4222ced413a9830c1e9e6a834e7257bfee0043d2f56166ef4aa1cb"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8904e865a2ac81d8160878e7788bc5ee6f4ca6948cf5c5198a83050d68537024"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c414ee94cd1986d7ea3700a6d80efc9ae9b194c37d77396bcfaf927b0d5a620e"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d5d2d1f4f35808a58c8fe7777db14526bd53f77a34b373f070912b2c23f2c3b"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ee1b84222c65f0e2b8ecb6362cc721df1953a0a59e13efc7a4055592fd897f8"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56450dee854ce494003f2be92f2eddb2531c02a456a7866dd32af467672c3b7b"}, + {file = "wassima-1.1.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28897780714f49331fd3e76531ea248df637bbf3e7bf4be175381a92d596c460"}, + {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7528cbfe710af7f9e92cd52296efd7a788466b7cc7fe575b196f604a6ba2281c"}, + {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_armv7l.whl", hash = "sha256:4c3325dff14e796d346e81f90067d054714b99a3d86b6d0a5a76d85bafd2b654"}, + {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:e6609ca3d620792c1dc137efff4c189adee0f13f266ae14515d7de2952159b95"}, + {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:60a695e750f9b4fc3cc79cbbbb5e507b8f9715e4135906bb1822126fad1ce5a2"}, + {file = "wassima-1.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:fca891820f7c679d3adc2443d6f85d6201db4badc6b17927d70fa498168d1aea"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:66efd9ee997bfb2311ade7a09f3174d6450a8695ab6b323840539c5826a276c6"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a8550eb00a31eac76a5b5fab3ca2e87cc8d91781191dffa3e133ebf574305321"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df677518d7779fc8a522132c4d96391e0a262dd12bb54ec3937bc8b58f6d3d5"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29d4f6d006ce96c2087c42414ad72ef71bc25bd20ac457dfe89ab2448b0d08e4"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e73264427c6e3f93c7e1b0529960a342a6b4c9c16d17785872a845ee2b0d28f5"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fdbc87751649a23377f279873aae902a38ce44162170edd6b6893d47a259a78"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce49ac61ca651f49c2664003215e259a017d5a1116d669ef331c4930214f53b0"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9522b4905fc75eeaac8518c54362e87d89e83bbefebdb1898a0ef025006e8241"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1b2e419d3075e425ecdcefd486ccd56697dc209e6e2120746477a995392b9402"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1ebe5b0feead8b0457b885f181156574bf9ca88df6fe4cef6ad6b364f02d9e98"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_i686.whl", hash = "sha256:6947c5e2d23383f00199b2cf638d7a090dfe5949bad113387e020b83f2423815"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4420be43f5b4e2b7721080130de565a582299d0d02771c9a7db55366d9c93da5"}, + {file = "wassima-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a9fb48413d80d41aa6531a2271516f63c8a1debac016cf8fad6a2fd30aa4486d"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9a4593db51fc02529950158f1217e08c9d62e1299e20a19858f07f80c6d09197"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-macosx_11_0_arm64.whl", hash = "sha256:127aecd895501e79b76114109dba0e4bcf6adcf47169f75d44ecd08b4d983ae7"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7d36eb4e92a348f58182f7f69b0e2fc680ac6605377f5201bac40b303727493"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0e29ace26e79b923d5b0f04c38dff44dc47b9c48684894d8f20841c6ee79760"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab77a0390ba74b7a011918ae5c187e2936cd46f4abffd37c5ff228dbdc4b5e89"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b18821d94eabef23946e21566e7ae7c009ef3a89fe1bc0204e791ba5fdb8ed5"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ef95ad25c04629939d6a6015a798c8b0435cebc0c53cc4b1dabb2a89214a4d8"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ebef47ce05288db306d4f56937f96c48da07afaec014a6ed46ecb17176f874bf"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f0833555f8a334cf1c824f24b07c6b01b13128825d16f7802c4c70d14d2dbe09"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:31afbbe4ea11ea9f92b152e4a61495614bfc0ae3d7c3215a24928144bab79f99"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_i686.whl", hash = "sha256:a759b84855b70922ee31234264ea2f4a058943a38270a18f00fd597f365b4bcb"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa01044ab3ca1f55e2d0d08128a97a68e9022795587627ee9edb3471c72e5df4"}, + {file = "wassima-1.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:08d6cf46441d73335b84c15c4f891adcb59f70701a13ecdee82aead5e0a9b134"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:23b56e0560bd2f35fceab001099bb890d8238fed64e7a0677cacbd1c4d870183"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac1866ee965263e3e024049044e8a5ce905fea2d40e005be03dcd89265fc1e6c"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4285c88f5cb4322318e3c3666d79b923f5317451efc2701011d960774d812675"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1f643f02b0824e789a7c98b9089dfd772a74ceec1a611cf420799f986cadb6bc"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9a7b91256ca429f99beff72dd89b0d5bd6ee1ca8f047138785c5b943eebfb1"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4224cf40a81840618a22164d4002fe5bb9b83cde957ec16e8913996809e705dd"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1006c7510b8495559fc2f1f27a7e49205140eb6b91a91f2c71cd91c2588522ae"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5abfa548d3c7acbf87899fc4af99c5a1fe929cf8cc7a7fd65a825dd88fa37b10"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c5854745eb0fd9243ebbaad46dc1f6f5193dd3f13f12dd19da95877ee2a8d62c"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:9def0580135d80a64aac4761e008d0d82fad5feb9c5028ba9427393144e4a535"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_i686.whl", hash = "sha256:450501472645fe5ea65f1848466ce5a0f2800ed5e13288fa4c210728e2883d24"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9764c226493e4a9b960156c3657ef7cece430ab8bad0035ebffb0eeb488633cb"}, + {file = "wassima-1.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:83792a234431f7fbd06f3370e968b99df430ab3bacdb9ea3318247d55dee3b6c"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:aee6fcfa43ce63691ec30943681e9432ff6cecbd976526c7ec0e5f2aaf85866a"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:31e69da1f3cf1ce4f24dbc4590101d68fcb3e1f715566fe30b6691429e9c1b10"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b574a646498b191bc8974524458d85bc55335992dc8ea7cddcb09ec58c01d4"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3112316434fd3ed3cfb1eac4998f54ed46d07a36172d18d543c0815a98e0d51"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:18e6f92114f878ea26fea7a10af255a6aadfddb1600f20fdfe96d65598e62501"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09d54c87ce23ec2332f2acefc030ae3f4262b94cb1f0c613c8d2e30c297d12d7"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9d953b261b7b64072fbed7b4bf4441f7910d8247387f29cc82f8c314f7acf39"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8ef5a3d129997147f5475c276bc79da14ac59a8f614f07634e2aac5d9b2f94"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e9a6da09d6a03c0c8ec3f5c6b7fa5061f051d67a0e0f0ec1518d2bd76efb7535"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:39d65b2beb0eb17a92cdf859d8e9146a15f8d7f35ab95602780a3ac078069e7e"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_i686.whl", hash = "sha256:e5ed0411e3a14e9352ff83e47952df03b7c8915f9fd4c9fb0888a80ac2759dcf"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:8f5869858975755d90e5505d3a7e2ac687cd09a348bc48137fd5b270969bd7a0"}, + {file = "wassima-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:39742c4884b7d1b3314064895e10345b96c7cab0a8f622e65f7beea89c0de4d2"}, + {file = "wassima-1.1.0-py3-none-any.whl", hash = "sha256:d250b77c1964c03f010a271fdd0cad3e14af250fb15cc3a729f23ee1e5922f69"}, + {file = "wassima-1.1.0.tar.gz", hash = "sha256:0ae03025ec07c0491e2d1a499d404eb66180c226f403451042190294f6ec7f06"}, +] + [[package]] name = "watchdog" version = "4.0.0" @@ -2613,4 +2824,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1532c2dc5846395a46766fead9f3c29223369ba11025b04e4eebec508fe0d8da" +content-hash = "38517d808d6bc20a9e2c8597b4024707537f2a92d1f75c67a5ed3477c139418b" diff --git a/pyproject.toml b/pyproject.toml index 53dfe2131..cb8766c38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ cachetools = "^5.3.2" socksio = "^1.0.0" jinja2 = "^3.1.3" pyzmq = "^25.1.2" +niquests = "^3.5.5" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From 84074c1e9cb99ed4a7dea69dcce6617f0126553d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 Apr 2024 17:45:49 -0400 Subject: [PATCH 119/171] WIP hook modules --- bbot/core/event/base.py | 6 +- bbot/core/helpers/dns/dns.py | 9 +-- bbot/core/helpers/dns/engine.py | 15 ---- bbot/modules/base.py | 35 +++++++--- bbot/modules/internal/dnsresolve.py | 103 ++++++++++++++++++++++------ bbot/scanner/manager.py | 95 ++++--------------------- bbot/scanner/scanner.py | 3 - 7 files changed, 128 insertions(+), 138 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index bf030ebc2..345e64115 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,6 +1,5 @@ import re import json -import asyncio import logging import ipaddress import traceback @@ -604,7 +603,7 @@ def json(self, mode="json", siem_friendly=False): j["scan"] = self.scan.id j["timestamp"] = self.timestamp.timestamp() if self.host: - j["resolved_hosts"] = [str(h) for h in self.resolved_hosts] + j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts) source_id = self.source_id if source_id: j["source"] = source_id @@ -951,7 +950,8 @@ def sanitize_data(self, data): @property def resolved_hosts(self): - return [".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")] + # TODO: remove this when we rip out httpx + return set(".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")) @property def pretty_string(self): diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index f9c505540..398842048 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -3,9 +3,7 @@ import dns.exception import dns.asyncresolver from cachetools import LRUCache -from contextlib import suppress -from bbot.errors import ValidationError from bbot.core.engine import EngineClient from bbot.core.helpers.async_helpers import NamedLock from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host @@ -96,7 +94,7 @@ async def resolve_event(self, event, minimal=False): # abort if the event doesn't have a host if (not event.host) or (event.type in ("IP_RANGE",)): # tags, whitelisted, blacklisted, children - return set(), False, False, dict() + return set(), dict() event_host = str(event.host) event_type = str(event.type) @@ -104,7 +102,7 @@ async def resolve_event(self, event, minimal=False): dns_children = dict() if (not event.host) or (event.type in ("IP_RANGE",)): - return event_tags, event_whitelisted, event_blacklisted, dns_children + return event_tags, dns_children # lock to ensure resolution of the same host doesn't start while we're working here async with self._event_cache_locks.lock(event_host): @@ -112,9 +110,6 @@ async def resolve_event(self, event, minimal=False): try: _event_tags, _dns_children = self._event_cache[event_host] event_tags.update(_event_tags) - # if we found it, return it - if _event_whitelisted is not None: - return event_tags, _dns_children except KeyError: pass diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 2b5903292..abba661a8 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -23,7 +23,6 @@ host_in_host, make_ip_type, smart_decode, - cloudcheck, rand_string, ) @@ -470,20 +469,6 @@ async def resolve_event(self, event_host, event_type, minimal=False): elif errors: event_tags.add(f"{rdtype.lower()}-error") - # tag with cloud providers - if not self.in_tests: - to_check = set() - if event_type == "IP_ADDRESS": - to_check.add(event_host) - for rdtype, ips in dns_children.items(): - if rdtype in ("A", "AAAA"): - for ip in ips: - to_check.add(ip) - for ip in to_check: - provider, provider_type, subnet = cloudcheck(ip) - if provider: - event_tags.add(f"{provider_type}-{provider}") - # if needed, mark as unresolved if not is_ip(event_host) and "resolved" not in event_tags: event_tags.add("unresolved") diff --git a/bbot/modules/base.py b/bbot/modules/base.py index a50203759..89db38fd4 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -683,6 +683,7 @@ def _event_precheck(self, event): if "target" not in event.tags: return False, "it did not meet target_only filter criteria" # exclude certain URLs (e.g. javascript): + # TODO: revisit this after httpx rework if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: return False, "its extension was listed in url_extension_httpx_only" @@ -1433,27 +1434,43 @@ async def _worker(self): if event.type == "FINISHED": context = f"{self.name}.finish()" async with self.scan._acatch(context), self._task_counter.count(context): - finish_task = asyncio.create_task(self.finish()) - await finish_task + await self.finish() continue self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") + + acceptable = True + async with self._task_counter.count(f"event_precheck({event})"): + precheck_pass, reason = self._event_precheck(event) + if not precheck_pass: + self.debug(f"Not hooking {event} because precheck failed ({reason})") + acceptable = False async with self._task_counter.count(f"event_postcheck({event})"): - acceptable, reason = await self._event_postcheck(event) + postcheck_pass, reason = await self._event_postcheck(event) + if not postcheck_pass: + self.debug(f"Not hooking {event} because postcheck failed ({reason})") + acceptable = False + + # whether to pass the event on to the rest of the scan + # defaults to true, unless handle_event returns False + pass_on_event = True + pass_on_event_reason = "" if acceptable: context = f"{self.name}.handle_event({event})" self.scan.stats.event_consumed(event, self) self.debug(f"Hooking {event}") async with self.scan._acatch(context), self._task_counter.count(context): - task_name = f"{self.name}.handle_event({event})" - handle_event_task = asyncio.create_task(self.handle_event(event), name=task_name) - await handle_event_task + pass_on_event = await self.handle_event(event) + with suppress(ValueError, TypeError): + pass_on_event, pass_on_event_reason = pass_on_event + self.debug(f"Finished hooking {event}") - else: - self.debug(f"Not hooking {event} because {reason}") - await self.outgoing_event_queue.put((event, _kwargs)) + if pass_on_event is False: + self.debug(f"Not passing on {event} because {pass_on_event_reason}") + else: + await self.outgoing_event_queue.put((event, _kwargs)) except asyncio.CancelledError: self.log.trace("Worker cancelled") diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 62ba4362a..13a0e9f76 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -1,26 +1,34 @@ +from contextlib import suppress + +from bbot.errors import ValidationError from bbot.modules.base import HookModule class dnsresolve(HookModule): - hooked_events = ["DNS_NAME"] + watched_events = ["*"] _priority = 1 async def setup(self): self.dns_resolution = self.scan.config.get("dns_resolution", False) + self.scope_search_distance = max(0, int(self.config.get("scope_search_distance", 0))) + self.scope_dns_search_distance = max(0, int(self.config.get("scope_dns_search_distance", 1))) + self.scope_distance_modifier = max(self.scope_search_distance, self.scope_dns_search_distance) return True - async def handle_event(self, event): - event.add_tag("dnsresolved") - resolved_hosts = set() - dns_children = {} - dns_tags = set() + async def filter_event(self, event): + if not event.host: + return False, "event does not have host attribute" + return True + async def handle_event(self, event): + self.hugesuccess(event) # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist # this is a performance optimization and it'd be nice if we could do it for all events not just targets # but for non-target events, we need to know what IPs they resolve to so we can make scope decisions about them skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist if skip_dns_resolution: dns_tags = {"resolved"} + dns_children = dict() else: # DNS resolution dns_tags, dns_children = await self.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) @@ -29,47 +37,102 @@ async def handle_event(self, event): event_whitelisted = False event_blacklisted = False for rdtype, children in dns_children.items(): + self.hugeinfo(f"{event.host}: {rdtype}:{children}") if event_blacklisted: break for host in children: if rdtype in ("A", "AAAA", "CNAME"): - for ip in ips: - resolved_hosts.add(ip) + event.resolved_hosts.add(host) # having a CNAME to an in-scope resource doesn't make you in-scope if not event_whitelisted and rdtype != "CNAME": with suppress(ValidationError): - if self.parent_helper.scan.whitelisted(host): + if self.scan.whitelisted(host): event_whitelisted = True # CNAME to a blacklisted resources, means you're blacklisted with suppress(ValidationError): - if self.parent_helper.scan.blacklisted(host): + if self.scan.blacklisted(host): event_blacklisted = True break # kill runaway DNS chains dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) if dns_resolve_distance >= self.helpers.dns.max_dns_resolve_distance: - log.debug( - f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.scan.helpers.dns.max_dns_resolve_distance})" + self.debug( + f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.max_dns_resolve_distance})" ) dns_children = {} if event.type in ("DNS_NAME", "IP_ADDRESS"): - event._dns_children = dns_children + event.dns_children = dns_children for tag in dns_tags: event.add_tag(tag) - event._resolved_hosts = resolved_hosts - - event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event) - event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event) if event_blacklisted: event.add_tag("blacklisted") reason = "event host" - if event_blacklisted_dns: + if event_blacklisted: reason = "DNS associations" - log.debug(f"Omitting due to blacklisted {reason}: {event}") + self.debug(f"Omitting due to blacklisted {reason}: {event}") return if event_whitelisted: - event.add_tag("whitelisted") + self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") + event.scope_distance = 0 + + # DNS_NAME --> DNS_NAME_UNRESOLVED + if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: + event.type = "DNS_NAME_UNRESOLVED" + + # check for wildcards + if event.scope_distance <= self.scan.scope_search_distance: + if not "unresolved" in event.tags: + if not self.helpers.is_ip_type(event.host): + await self.helpers.dns.handle_wildcard_event(event) + + # speculate DNS_NAMES and IP_ADDRESSes from other event types + source_event = event + if ( + event.host + and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") + and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") + ): + source_event = self.make_event(event.host, "DNS_NAME", source=event) + # only emit the event if it's not already in the parent chain + if source_event is not None and source_event not in event.get_sources(): + source_event.scope_distance = event.scope_distance + if "target" in event.tags: + source_event.add_tag("target") + await self.emit_event(source_event) + + ### Emit DNS children ### + if self.dns_resolution: + self.hugesuccess(f"emitting children for {event}! (dns children: {event.dns_children})") + emit_children = True + in_dns_scope = -1 < event.scope_distance < self.scope_distance_modifier + self.critical(f"{event.host} in dns scope: {in_dns_scope}") + + if emit_children: + dns_child_events = [] + if event.dns_children: + for rdtype, records in event.dns_children.items(): + self.hugewarning(f"{event.host}: {rdtype}:{records}") + module = self.scan._make_dummy_module_dns(rdtype) + module._priority = 4 + for record in records: + try: + child_event = self.scan.make_event( + record, "DNS_NAME", module=module, source=source_event + ) + # if it's a hostname and it's only one hop away, mark it as affiliate + if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: + child_event.add_tag("affiliate") + host_hash = hash(str(child_event.host)) + if in_dns_scope or self.preset.in_scope(child_event): + dns_child_events.append(child_event) + except ValidationError as e: + self.warning( + f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' + ) + for child_event in dns_child_events: + self.debug(f"Queueing DNS child for {event}: {child_event}") + await self.emit_event(child_event) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 4d6a0e239..df56361cc 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -62,14 +62,10 @@ async def _worker_loop(self): # if we have hooks set up, we always get events from the last (lowest priority) hook module. if self.hook_modules: last_hook_module = self.hook_modules[-1] - incoming = last_hook_module.outgoing_event_queue.get_nowait() + event, kwargs = last_hook_module.outgoing_event_queue.get_nowait() else: # otherwise, we go through all the modules - incoming = self.get_event_from_modules() - try: - event, kwargs = incoming - except: - log.critical(incoming) + event, kwargs = self.get_event_from_modules() except asyncio.queues.QueueEmpty: await asyncio.sleep(0.1) continue @@ -131,6 +127,8 @@ async def emit_event(self, event, *args, **kwargs): # skip event if it fails precheck if event.type != "DNS_NAME": acceptable = self._event_precheck(event) + if not acceptable: + return log.debug(f'Module "{event.module}" raised {event}') @@ -204,38 +202,25 @@ async def _emit_event(self, event, **kwargs): on_success_callback = kwargs.pop("on_success_callback", None) abort_if = kwargs.pop("abort_if", None) - event_whitelisted = "whitelisted" in event.tags - - # other blacklist rejections - URL extensions, etc. - if "blacklisted" in event.tags: + # blacklist rejections + event_blacklisted = self.scan.blacklisted(event) + if event_blacklisted or "blacklisted" in event.tags: log.debug(f"Omitting blacklisted event: {event}") return - # DNS_NAME --> DNS_NAME_UNRESOLVED - if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: - event.type = "DNS_NAME_UNRESOLVED" - - # Cloud tagging - await self.scan.helpers.cloud.tag_event(event) - - # Scope shepherding - # here is where we make sure in-scope events are set to their proper scope distance - if event.host and event_whitelisted: - log.debug(f"Making {event} in-scope") - event.scope_distance = 0 - - # check for wildcards - if event.scope_distance <= self.scan.scope_search_distance: - if not "unresolved" in event.tags: - if not self.scan.helpers.is_ip_type(event.host): - await self.scan.helpers.dns.handle_wildcard_event(event) - # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() if event.type == "DNS_NAME": acceptable = self._event_precheck(event) if not acceptable: return + # Scope shepherding + # here is where we make sure in-scope events are set to their proper scope distance + event_whitelisted = self.scan.whitelisted(event) + if event.host and event_whitelisted: + log.debug(f"Making {event} in-scope because it matches the scan target") + event.scope_distance = 0 + # now that the event is properly tagged, we can finally make decisions about it abort_result = False if callable(abort_if): @@ -256,58 +241,6 @@ async def _emit_event(self, event, **kwargs): await self.distribute_event(event) - # speculate DNS_NAMES and IP_ADDRESSes from other event types - source_event = event - if ( - event.host - and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") - and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") - ): - source_module = self.scan._make_dummy_module("host", _type="internal") - source_module._priority = 4 - source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event) - # only emit the event if it's not already in the parent chain - if source_event is not None and source_event not in source_event.get_sources(): - source_event.scope_distance = event.scope_distance - if "target" in event.tags: - source_event.add_tag("target") - self.queue_event(source_event) - - ### Emit DNS children ### - if self.dns_resolution: - emit_children = True - in_dns_scope = -1 < event.scope_distance < self.scan.scope_dns_search_distance - # only emit DNS children once for each unique host - host_hash = hash(str(event.host)) - if host_hash in self.outgoing_dup_tracker: - emit_children = False - self.outgoing_dup_tracker.add(host_hash) - - if emit_children: - dns_child_events = [] - if event.dns_children: - for rdtype, records in event.dns_children.items(): - module = self.scan._make_dummy_module_dns(rdtype) - module._priority = 4 - for record in records: - try: - child_event = self.scan.make_event( - record, "DNS_NAME", module=module, source=source_event - ) - # if it's a hostname and it's only one hop away, mark it as affiliate - if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: - child_event.add_tag("affiliate") - host_hash = hash(str(child_event.host)) - if in_dns_scope or self.preset.in_scope(child_event): - dns_child_events.append(child_event) - except ValidationError as e: - log.warning( - f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' - ) - for child_event in dns_child_events: - log.debug(f"Queueing DNS child for {event}: {child_event}") - self.queue_event(child_event) - except ValidationError as e: log.warning(f"Event validation failed with kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index e2b813cbc..488446103 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -182,9 +182,6 @@ def __init__( # scope distance self.scope_search_distance = max(0, int(self.config.get("scope_search_distance", 0))) - self.scope_dns_search_distance = max( - self.scope_search_distance, int(self.config.get("scope_dns_search_distance", 1)) - ) self.scope_report_distance = int(self.config.get("scope_report_distance", 1)) # url file extensions From 07a4cbdbde5cd38539d6888be2d79615d26125ca Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 9 Apr 2024 02:11:05 -0400 Subject: [PATCH 120/171] continued work on hooks --- bbot/core/helpers/dns/dns.py | 36 ------- bbot/core/helpers/dns/engine.py | 20 +--- bbot/defaults.yml | 9 +- bbot/modules/internal/dnsresolve.py | 146 +++++++++++++++------------- bbot/scanner/manager.py | 13 +-- bbot/scanner/scanner.py | 2 +- 6 files changed, 96 insertions(+), 130 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 398842048..821d7b667 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -5,7 +5,6 @@ from cachetools import LRUCache from bbot.core.engine import EngineClient -from bbot.core.helpers.async_helpers import NamedLock from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host from .engine import DNSEngine @@ -35,7 +34,6 @@ class DNSHelper(EngineClient): wildcard_tests (int): Number of tests to be run for wildcard detection. Defaults to 5. _wildcard_cache (dict): Cache for wildcard detection results. _dns_cache (LRUCache): Cache for DNS resolution results, limited in size. - _event_cache (LRUCache): Cache for event resolution results, tags. Limited in size. resolver_file (Path): File containing system's current resolver nameservers. filter_bad_ptrs (bool): Whether to filter out DNS names that appear to be auto-generated PTR records. Defaults to True. @@ -70,10 +68,6 @@ def __init__(self, parent_helper): self.wildcard_ignore = [] self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) - # event resolution cache - self._event_cache = LRUCache(maxsize=10000) - self._event_cache_locks = NamedLock() - # copy the system's current resolvers to a text file for tool use self.system_resolvers = dns.resolver.Resolver().nameservers # TODO: DNS server speed test (start in background task) @@ -90,36 +84,6 @@ async def resolve_raw_batch(self, queries): async for _ in self.run_and_yield("resolve_raw_batch", queries=queries): yield _ - async def resolve_event(self, event, minimal=False): - # abort if the event doesn't have a host - if (not event.host) or (event.type in ("IP_RANGE",)): - # tags, whitelisted, blacklisted, children - return set(), dict() - - event_host = str(event.host) - event_type = str(event.type) - event_tags = set() - dns_children = dict() - - if (not event.host) or (event.type in ("IP_RANGE",)): - return event_tags, dns_children - - # lock to ensure resolution of the same host doesn't start while we're working here - async with self._event_cache_locks.lock(event_host): - # try to get data from cache - try: - _event_tags, _dns_children = self._event_cache[event_host] - event_tags.update(_event_tags) - except KeyError: - pass - - kwargs = {"event_host": event_host, "event_type": event_type, "minimal": minimal} - event_tags, dns_children = await self.run_and_return("resolve_event", **kwargs) - - self._event_cache[event_host] = (event_tags, dns_children) - - return event_tags, dns_children - async def is_wildcard(self, query, ips=None, rdtype=None): """ Use this method to check whether a *host* is a wildcard entry diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index abba661a8..491b433cd 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -29,6 +29,8 @@ log = logging.getLogger("bbot.core.helpers.dns.engine.server") +all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] + class DNSEngine(EngineServer): @@ -42,8 +44,6 @@ class DNSEngine(EngineServer): 99: "_mock_dns", } - all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] - def __init__(self, socket_path, config={}): super().__init__(socket_path) @@ -443,7 +443,7 @@ async def resolve_event(self, event_host, event_type, minimal=False): types = ("PTR",) else: if event_type == "DNS_NAME" and not minimal: - types = self.all_rdtypes + types = all_rdtypes else: types = ("A", "AAAA") queries = [(event_host, t) for t in types] @@ -708,7 +708,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): parent = parent_domain(query) parents = list(domain_parents(query)) - rdtypes_to_check = [rdtype] if rdtype is not None else self.all_rdtypes + rdtypes_to_check = [rdtype] if rdtype is not None else all_rdtypes query_baseline = dict() # if the caller hasn't already done the work of resolving the IPs @@ -807,7 +807,7 @@ async def is_wildcard_domain(self, domain, log_info=False): log.debug(f"Skipping wildcard detection on {domain} because it is excluded in the config") return {} - rdtypes_to_check = set(self.all_rdtypes) + rdtypes_to_check = all_rdtypes # make a list of its parents parents = list(domain_parents(domain, include_self=True)) @@ -890,16 +890,6 @@ async def _connectivity_check(self, interval=5): self._errors.clear() return False - def _parse_rdtype(self, t, default=None): - if isinstance(t, str): - if t.strip().lower() in ("any", "all", "*"): - return self.all_rdtypes - else: - return [t.strip().upper()] - elif any([isinstance(t, x) for x in (list, tuple)]): - return [str(_).strip().upper() for _ in t] - return default - def debug(self, *args, **kwargs): if self._debug: log.trace(*args, **kwargs) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 6255f0b71..5b6323ae4 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -8,8 +8,6 @@ home: ~/.bbot scope_report_distance: 0 # Generate new DNS_NAME and IP_ADDRESS events through DNS resolution dns_resolution: true -# Limit the number of BBOT threads -max_threads: 25 # Rate-limit DNS dns_queries_per_second: 1000 # Rate-limit HTTP @@ -42,12 +40,19 @@ scope_dns_search_distance: 1 # Limit how many DNS records can be followed in a row (stop malicious/runaway DNS records) dns_resolve_distance: 5 +# Limit the number of scan manager workers +manager_tasks: 5 + # Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc. speculate: True # Passively search event data for URLs, hostnames, emails, etc. excavate: True # Summarize activity at the end of a scan aggregate: True +# DNS resolution +dnsresolve: True +# Cloud provider tagging +cloudcheck: True # How to handle installation of module dependencies # Choices are: diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 13a0e9f76..dc8ba996b 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -1,58 +1,81 @@ from contextlib import suppress +from cachetools import LRUCache from bbot.errors import ValidationError from bbot.modules.base import HookModule +from bbot.core.helpers.dns.engine import all_rdtypes +from bbot.core.helpers.async_helpers import NamedLock class dnsresolve(HookModule): watched_events = ["*"] _priority = 1 + _max_event_handlers = 25 async def setup(self): self.dns_resolution = self.scan.config.get("dns_resolution", False) - self.scope_search_distance = max(0, int(self.config.get("scope_search_distance", 0))) - self.scope_dns_search_distance = max(0, int(self.config.get("scope_dns_search_distance", 1))) - self.scope_distance_modifier = max(self.scope_search_distance, self.scope_dns_search_distance) + self.scope_search_distance = max(0, int(self.scan.config.get("scope_search_distance", 0))) + self.scope_dns_search_distance = max(0, int(self.scan.config.get("scope_dns_search_distance", 1))) + # event resolution cache + self._event_cache = LRUCache(maxsize=10000) + self._event_cache_locks = NamedLock() return True + @property + def scope_distance_modifier(self): + return max(self.scope_search_distance, self.scope_dns_search_distance) + async def filter_event(self, event): - if not event.host: + if (not event.host) or (event.type in ("IP_RANGE",)): return False, "event does not have host attribute" return True async def handle_event(self, event): - self.hugesuccess(event) - # skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist - # this is a performance optimization and it'd be nice if we could do it for all events not just targets - # but for non-target events, we need to know what IPs they resolve to so we can make scope decisions about them - skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist - if skip_dns_resolution: - dns_tags = {"resolved"} - dns_children = dict() - else: - # DNS resolution - dns_tags, dns_children = await self.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) - - # whitelisting / blacklisting based on resolved hosts + dns_tags = set() + dns_children = dict() + + # DNS resolution event_whitelisted = False event_blacklisted = False - for rdtype, children in dns_children.items(): - self.hugeinfo(f"{event.host}: {rdtype}:{children}") - if event_blacklisted: - break - for host in children: - if rdtype in ("A", "AAAA", "CNAME"): - event.resolved_hosts.add(host) - # having a CNAME to an in-scope resource doesn't make you in-scope - if not event_whitelisted and rdtype != "CNAME": - with suppress(ValidationError): - if self.scan.whitelisted(host): - event_whitelisted = True - # CNAME to a blacklisted resources, means you're blacklisted - with suppress(ValidationError): - if self.scan.blacklisted(host): - event_blacklisted = True - break + + event_host = str(event.host) + event_host_hash = hash(str(event.host)) + + emit_children = event_host_hash not in self._event_cache + + async with self._event_cache_locks.lock(event_host_hash): + try: + # try to get from cache + dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] + except KeyError: + queries = [(event_host, rdtype) for rdtype in all_rdtypes] + async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): + for answer, _rdtype in answers: + dns_tags.add(f"{rdtype.lower()}-record") + try: + dns_children[_rdtype].add(answer) + except KeyError: + dns_children[_rdtype] = {answer} + + # whitelisting / blacklisting based on resolved hosts + for rdtype, children in dns_children.items(): + if event_blacklisted: + break + for host in children: + if rdtype in ("A", "AAAA", "CNAME"): + event.resolved_hosts.add(host) + # having a CNAME to an in-scope resource doesn't make you in-scope + if not event_whitelisted and rdtype != "CNAME": + with suppress(ValidationError): + if self.scan.whitelisted(host): + event_whitelisted = True + # CNAME to a blacklisted resources, means you're blacklisted + with suppress(ValidationError): + if self.scan.blacklisted(host): + event_blacklisted = True + break + + self._event_cache[event_host_hash] = dns_tags, dns_children, event_whitelisted, event_blacklisted # kill runaway DNS chains dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) @@ -73,7 +96,6 @@ async def handle_event(self, event): if event_blacklisted: reason = "DNS associations" self.debug(f"Omitting due to blacklisted {reason}: {event}") - return if event_whitelisted: self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") @@ -102,37 +124,29 @@ async def handle_event(self, event): source_event.scope_distance = event.scope_distance if "target" in event.tags: source_event.add_tag("target") - await self.emit_event(source_event) + self.scan.manager.queue_event(source_event) ### Emit DNS children ### - if self.dns_resolution: - self.hugesuccess(f"emitting children for {event}! (dns children: {event.dns_children})") - emit_children = True + if emit_children: in_dns_scope = -1 < event.scope_distance < self.scope_distance_modifier - self.critical(f"{event.host} in dns scope: {in_dns_scope}") - - if emit_children: - dns_child_events = [] - if event.dns_children: - for rdtype, records in event.dns_children.items(): - self.hugewarning(f"{event.host}: {rdtype}:{records}") - module = self.scan._make_dummy_module_dns(rdtype) - module._priority = 4 - for record in records: - try: - child_event = self.scan.make_event( - record, "DNS_NAME", module=module, source=source_event - ) - # if it's a hostname and it's only one hop away, mark it as affiliate - if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: - child_event.add_tag("affiliate") - host_hash = hash(str(child_event.host)) - if in_dns_scope or self.preset.in_scope(child_event): - dns_child_events.append(child_event) - except ValidationError as e: - self.warning( - f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' - ) - for child_event in dns_child_events: - self.debug(f"Queueing DNS child for {event}: {child_event}") - await self.emit_event(child_event) + dns_child_events = [] + if event.dns_children: + for rdtype, records in event.dns_children.items(): + module = self.scan._make_dummy_module_dns(rdtype) + module._priority = 4 + for record in records: + try: + child_event = self.scan.make_event(record, "DNS_NAME", module=module, source=source_event) + # if it's a hostname and it's only one hop away, mark it as affiliate + if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: + child_event.add_tag("affiliate") + host_hash = hash(str(child_event.host)) + if in_dns_scope or self.preset.in_scope(child_event): + dns_child_events.append(child_event) + except ValidationError as e: + self.warning( + f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' + ) + for child_event in dns_child_events: + self.debug(f"Queueing DNS child for {event}: {child_event}") + self.scan.manager.queue_event(child_event) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index df56361cc..45887a05d 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -125,10 +125,9 @@ async def emit_event(self, event, *args, **kwargs): quick = (quick_kwarg or quick_event) and not callbacks_requested # skip event if it fails precheck - if event.type != "DNS_NAME": - acceptable = self._event_precheck(event) - if not acceptable: - return + acceptable = self._event_precheck(event) + if not acceptable: + return log.debug(f'Module "{event.module}" raised {event}') @@ -208,12 +207,6 @@ async def _emit_event(self, event, **kwargs): log.debug(f"Omitting blacklisted event: {event}") return - # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() - if event.type == "DNS_NAME": - acceptable = self._event_precheck(event) - if not acceptable: - return - # Scope shepherding # here is where we make sure in-scope events are set to their proper scope distance event_whitelisted = self.scan.whitelisted(event) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 488446103..42a199d57 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -165,7 +165,7 @@ def __init__( self._status = "NOT_STARTED" self._status_code = 0 - self.max_workers = max(1, self.config.get("max_threads", 25)) + self.max_workers = max(1, self.config.get("manager_tasks", 5)) self.modules = OrderedDict({}) self._modules_loaded = False From b1a8c2329011930382ffbf01100d89b831b6d348 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 9 Apr 2024 02:29:39 -0400 Subject: [PATCH 121/171] more work on hooks --- bbot/core/helpers/dns/engine.py | 79 ----------------------------- bbot/modules/internal/dnsresolve.py | 25 ++++++++- 2 files changed, 24 insertions(+), 80 deletions(-) diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 491b433cd..5e5f2b81d 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -408,85 +408,6 @@ async def handle_wildcard_event(self, event, children): finally: log.debug(f"Finished handle_wildcard_event({event}, children={children})") - async def resolve_event(self, event_host, event_type, minimal=False): - """ - Tag the given event with the appropriate DNS record types and optionally create child - events based on DNS resolutions. - - Args: - event (object): The event to be resolved and tagged. - minimal (bool, optional): If set to True, the function will perform minimal DNS - resolution. Defaults to False. - - Returns: - tuple: A 4-tuple containing the following items: - - event_tags (set): Set of tags for the event. - - dns_children (dict): Dictionary containing child events from DNS resolutions. - - Examples: - >>> event = make_event("evilcorp.com") - >>> resolve_event(event) - ({'resolved', 'ns-record', 'a-record',}, False, False, {'A': {IPv4Address('1.2.3.4'), IPv4Address('1.2.3.5')}, 'NS': {'ns1.evilcorp.com'}}) - - Note: - This method does not modify the passed in `event`. Instead, it returns data - that can be used to modify or act upon the `event`. - """ - log.debug(f"Resolving event {event_type}:{event_host}") - event_tags = set() - dns_children = dict() - - try: - types = () - if is_ip(event_host): - if not minimal: - types = ("PTR",) - else: - if event_type == "DNS_NAME" and not minimal: - types = all_rdtypes - else: - types = ("A", "AAAA") - queries = [(event_host, t) for t in types] - async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(queries): - if answers: - rdtype = str(rdtype).upper() - event_tags.add("resolved") - event_tags.add(f"{rdtype.lower()}-record") - - for host, _rdtype in answers: - if host: - host = make_ip_type(host) - - if self.filter_bad_ptrs and rdtype in ("PTR") and is_ptr(host): - self.debug(f"Filtering out bad PTR: {host}") - continue - - try: - dns_children[_rdtype].add(host) - except KeyError: - dns_children[_rdtype] = {host} - - elif errors: - event_tags.add(f"{rdtype.lower()}-error") - - # if needed, mark as unresolved - if not is_ip(event_host) and "resolved" not in event_tags: - event_tags.add("unresolved") - # check for private IPs - for rdtype, ips in dns_children.items(): - for ip in ips: - try: - ip = ipaddress.ip_address(ip) - if ip.is_private: - event_tags.add("private-ip") - except ValueError: - continue - - return event_tags, dns_children - - finally: - log.debug(f"Finished resolving event {event_type}:{event_host}") - def event_cache_get(self, host): """ Retrieves cached event data based on the given host. diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index dc8ba996b..f7295235e 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -1,3 +1,4 @@ +import ipaddress from contextlib import suppress from cachetools import LRUCache @@ -40,6 +41,7 @@ async def handle_event(self, event): event_host = str(event.host) event_host_hash = hash(str(event.host)) + event_is_ip = self.helpers.is_ip(event.host) emit_children = event_host_hash not in self._event_cache @@ -49,7 +51,10 @@ async def handle_event(self, event): dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] except KeyError: queries = [(event_host, rdtype) for rdtype in all_rdtypes] + error_rdtypes = [] async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): + if errors: + error_rdtypes.append(rdtype) for answer, _rdtype in answers: dns_tags.add(f"{rdtype.lower()}-record") try: @@ -57,11 +62,21 @@ async def handle_event(self, event): except KeyError: dns_children[_rdtype] = {answer} - # whitelisting / blacklisting based on resolved hosts + for rdtype in error_rdtypes: + if rdtype not in dns_children: + dns_tags.add(f"{rdtype.lower()}-error") + + if not event_is_ip: + if dns_children: + dns_tags.add("resolved") + else: + dns_tags.add("unresolved") + for rdtype, children in dns_children.items(): if event_blacklisted: break for host in children: + # whitelisting / blacklisting based on resolved hosts if rdtype in ("A", "AAAA", "CNAME"): event.resolved_hosts.add(host) # having a CNAME to an in-scope resource doesn't make you in-scope @@ -75,6 +90,14 @@ async def handle_event(self, event): event_blacklisted = True break + # check for private IPs + try: + ip = ipaddress.ip_address(host) + if ip.is_private: + dns_tags.add("private-ip") + except ValueError: + continue + self._event_cache[event_host_hash] = dns_tags, dns_children, event_whitelisted, event_blacklisted # kill runaway DNS chains From 490bd36a0896b517e582aeb824d2bb8194ac1ab3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 9 Apr 2024 15:46:26 -0400 Subject: [PATCH 122/171] more wip hooks --- bbot/core/helpers/dns/dns.py | 102 ++++---------------- bbot/core/helpers/dns/engine.py | 140 +++------------------------- bbot/modules/base.py | 9 +- bbot/modules/internal/dnsresolve.py | 107 +++++++++++++++------ bbot/scanner/manager.py | 2 +- bbot/test/test_step_1/test_dns.py | 88 +++++++++++------ 6 files changed, 180 insertions(+), 268 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 821d7b667..cc0a1ff4a 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -2,7 +2,6 @@ import logging import dns.exception import dns.asyncresolver -from cachetools import LRUCache from bbot.core.engine import EngineClient from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host @@ -119,20 +118,10 @@ async def is_wildcard(self, query, ips=None, rdtype=None): if [ips, rdtype].count(None) == 1: raise ValueError("Both ips and rdtype must be specified") - # skip if query isn't a dns name - if not is_dns_name(query): + query = self._wildcard_prevalidation(query) + if not query: return {} - # skip check if the query's parent domain is excluded in the config - for d in self.wildcard_ignore: - if host_in_host(query, d): - log.debug(f"Skipping wildcard detection on {query} because it is excluded in the config") - return {} - - query = clean_dns_record(query) - # skip check if it's an IP or a plain hostname - if is_ip(query) or not "." in query: - return {} # skip check if the query is a domain if is_domain(query): return {} @@ -140,80 +129,29 @@ async def is_wildcard(self, query, ips=None, rdtype=None): return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype) async def is_wildcard_domain(self, domain, log_info=False): - return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) + domain = self._wildcard_prevalidation(domain) + if not domain: + return {} - async def handle_wildcard_event(self, event): - """ - Used within BBOT's scan manager to detect and tag DNS wildcard events. + return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) - Wildcards are detected for every major record type. If a wildcard is detected, its data - is overwritten, for example: `_wildcard.evilcorp.com`. + def _wildcard_prevalidation(self, host): + host = clean_dns_record(host) + # skip check if it's an IP or a plain hostname + if is_ip(host) or not "." in host: + return False - Args: - event (Event): The event to check for wildcards. + # skip if query isn't a dns name + if not is_dns_name(host): + return False - Returns: - None: This method modifies the `event` in place and does not return a value. + # skip check if the query's parent domain is excluded in the config + for d in self.wildcard_ignore: + if host_in_host(host, d): + log.debug(f"Skipping wildcard detection on {host} because it is excluded in the config") + return False - Examples: - >>> handle_wildcard_event(event) - # The `event` might now have tags like ["wildcard", "a-wildcard", "aaaa-wildcard"] and - # its `data` attribute might be modified to "_wildcard.evilcorp.com" if it was detected - # as a wildcard. - """ - log.debug(f"Entering handle_wildcard_event({event}, children={event.dns_children})") - try: - event_host = str(event.host) - # wildcard checks - if not is_ip(event.host): - # check if the dns name itself is a wildcard entry - wildcard_rdtypes = await self.is_wildcard(event_host) - for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): - wildcard_tag = "error" - if is_wildcard == True: - event.add_tag("wildcard") - wildcard_tag = "wildcard" - event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") - - # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if (not is_ip(event.host)) and event.dns_children: - if wildcard_rdtypes: - # these are the rdtypes that successfully resolve - resolved_rdtypes = set([c.upper() for c in event.dns_children]) - # these are the rdtypes that have wildcards - wildcard_rdtypes_set = set(wildcard_rdtypes) - # consider the event a full wildcard if all its records are wildcards - event_is_wildcard = False - if resolved_rdtypes: - event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) - - if event_is_wildcard: - if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = self.parent_helper.parent_domain(event_host) - for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): - if _is_wildcard: - wildcard_parent = _parent_domain - break - wildcard_data = f"_wildcard.{wildcard_parent}" - if wildcard_data != event.data: - log.debug( - f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"' - ) - event.data = wildcard_data - - # TODO: transplant this - # tag wildcard domains for convenience - # elif is_domain(event_host) or hash(event_host) in self._wildcard_cache: - # event_target = "target" in event.tags - # wildcard_domain_results = await self.is_wildcard_domain(event_host, log_info=event_target) - # for hostname, wildcard_domain_rdtypes in wildcard_domain_results.items(): - # if wildcard_domain_rdtypes: - # event.add_tag("wildcard-domain") - # for rdtype, ips in wildcard_domain_rdtypes.items(): - # event.add_tag(f"{rdtype.lower()}-wildcard-domain") - - finally: - log.debug(f"Finished handle_wildcard_event({event}, children={event.dns_children})") + return host async def _mock_dns(self, mock_data): from .mock import MockResolver diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 5e5f2b81d..b8e184264 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -3,27 +3,21 @@ import time import asyncio import logging -import ipaddress import traceback -from contextlib import suppress from cachetools import LRUCache +from contextlib import suppress from ..regexes import dns_name_regex from bbot.errors import DNSWildcardBreak from bbot.core.engine import EngineServer from bbot.core.helpers.async_helpers import NamedLock from bbot.core.helpers.misc import ( - clean_dns_record, - parent_domain, - domain_parents, is_ip, - is_domain, - is_ptr, - is_dns_name, - host_in_host, - make_ip_type, - smart_decode, rand_string, + smart_decode, + parent_domain, + domain_parents, + clean_dns_record, ) @@ -36,11 +30,10 @@ class DNSEngine(EngineServer): CMDS = { 0: "resolve", - 1: "resolve_event", - 2: "resolve_batch", - 3: "resolve_raw_batch", - 4: "is_wildcard", - 5: "is_wildcard_domain", + 1: "resolve_batch", + 2: "resolve_raw_batch", + 3: "is_wildcard", + 4: "is_wildcard_domain", 99: "_mock_dns", } @@ -336,107 +329,6 @@ async def _resolve_ip(self, query, **kwargs): return results, errors - async def handle_wildcard_event(self, event, children): - """ - Used within BBOT's scan manager to detect and tag DNS wildcard events. - - Wildcards are detected for every major record type. If a wildcard is detected, its data - is overwritten, for example: `_wildcard.evilcorp.com`. - - Args: - event (object): The event to check for wildcards. - children (list): A list of the event's resulting DNS children after resolution. - - Returns: - None: This method modifies the `event` in place and does not return a value. - - Examples: - >>> handle_wildcard_event(event, children) - # The `event` might now have tags like ["wildcard", "a-wildcard", "aaaa-wildcard"] and - # its `data` attribute might be modified to "_wildcard.evilcorp.com" if it was detected - # as a wildcard. - """ - log.debug(f"Entering handle_wildcard_event({event}, children={children})") - try: - event_host = str(event.host) - event_is_ip = is_ip(event_host) - # wildcard checks - if not event_is_ip: - # check if the dns name itself is a wildcard entry - wildcard_rdtypes = await self.is_wildcard(event_host) - for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): - wildcard_tag = "error" - if is_wildcard == True: - event.add_tag("wildcard") - wildcard_tag = "wildcard" - event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") - - # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if not event_is_ip and children: - if wildcard_rdtypes: - # these are the rdtypes that successfully resolve - resolved_rdtypes = set([c.upper() for c in children]) - # these are the rdtypes that have wildcards - wildcard_rdtypes_set = set(wildcard_rdtypes) - # consider the event a full wildcard if all its records are wildcards - event_is_wildcard = False - if resolved_rdtypes: - event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) - - if event_is_wildcard: - if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = parent_domain(event_host) - for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): - if _is_wildcard: - wildcard_parent = _parent_domain - break - wildcard_data = f"_wildcard.{wildcard_parent}" - if wildcard_data != event.data: - log.debug( - f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"' - ) - event.data = wildcard_data - # tag wildcard domains for convenience - elif is_domain(event_host) or hash(event_host) in self._wildcard_cache: - event_target = "target" in event.tags - wildcard_domain_results = await self.is_wildcard_domain(event_host, log_info=event_target) - for hostname, wildcard_domain_rdtypes in wildcard_domain_results.items(): - if wildcard_domain_rdtypes: - event.add_tag("wildcard-domain") - for rdtype, ips in wildcard_domain_rdtypes.items(): - event.add_tag(f"{rdtype.lower()}-wildcard-domain") - finally: - log.debug(f"Finished handle_wildcard_event({event}, children={children})") - - def event_cache_get(self, host): - """ - Retrieves cached event data based on the given host. - - Args: - host (str): The host for which the event data is to be retrieved. - - Returns: - tuple: A 4-tuple containing the following items: - - event_tags (set): Set of tags for the event. - - dns_children (set): Set containing child events from DNS resolutions. - - Examples: - Assuming an event with host "www.evilcorp.com" has been cached: - - >>> event_cache_get("www.evilcorp.com") - ({"resolved", "a-record"}, False, False, {'1.2.3.4'}) - - Assuming no event with host "www.notincache.com" has been cached: - - >>> event_cache_get("www.notincache.com") - (set(), set()) - """ - try: - event_tags, dns_children = self._event_cache[host] - return (event_tags, dns_children) - except KeyError: - return set(), set() - async def resolve_batch(self, queries, threads=10, **kwargs): """ A helper to execute a bunch of DNS requests. @@ -717,18 +609,8 @@ async def is_wildcard_domain(self, domain, log_info=False): {} """ wildcard_domain_results = {} - domain = clean_dns_record(domain) - - if not is_dns_name(domain): - return {} - - # skip check if the query's parent domain is excluded in the config - for d in self.wildcard_ignore: - if host_in_host(domain, d): - log.debug(f"Skipping wildcard detection on {domain} because it is excluded in the config") - return {} - rdtypes_to_check = all_rdtypes + rdtypes_to_check = set(all_rdtypes) # make a list of its parents parents = list(domain_parents(domain, include_self=True)) @@ -751,7 +633,7 @@ async def is_wildcard_domain(self, domain, log_info=False): wildcard_results = dict() queries = [] - for rdtype in list(rdtypes_to_check): + for rdtype in rdtypes_to_check: for _ in range(self.wildcard_tests): rand_query = f"{rand_string(digits=False, length=10)}.{host}" queries.append((rand_query, rdtype)) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 89db38fd4..52d52d656 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1467,10 +1467,11 @@ async def _worker(self): self.debug(f"Finished hooking {event}") - if pass_on_event is False: - self.debug(f"Not passing on {event} because {pass_on_event_reason}") - else: - await self.outgoing_event_queue.put((event, _kwargs)) + if pass_on_event is False: + self.debug(f"Not passing on {event} because {pass_on_event_reason}") + return + + await self.outgoing_event_queue.put((event, _kwargs)) except asyncio.CancelledError: self.log.trace("Worker cancelled") diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index f7295235e..9ff52e15e 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -17,9 +17,11 @@ async def setup(self): self.dns_resolution = self.scan.config.get("dns_resolution", False) self.scope_search_distance = max(0, int(self.scan.config.get("scope_search_distance", 0))) self.scope_dns_search_distance = max(0, int(self.scan.config.get("scope_dns_search_distance", 1))) + # event resolution cache self._event_cache = LRUCache(maxsize=10000) self._event_cache_locks = NamedLock() + return True @property @@ -34,8 +36,6 @@ async def filter_event(self, event): async def handle_event(self, event): dns_tags = set() dns_children = dict() - - # DNS resolution event_whitelisted = False event_blacklisted = False @@ -43,13 +43,17 @@ async def handle_event(self, event): event_host_hash = hash(str(event.host)) event_is_ip = self.helpers.is_ip(event.host) + # only emit DNS children if we haven't seen this host before emit_children = event_host_hash not in self._event_cache + # we do DNS resolution inside a lock to make sure we don't duplicate work + # once the resolution happens, it will be cached so it doesn't need to happen again async with self._event_cache_locks.lock(event_host_hash): try: # try to get from cache dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] except KeyError: + # if missing from cache, do DNS resolution queries = [(event_host, rdtype) for rdtype in all_rdtypes] error_rdtypes = [] async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): @@ -78,7 +82,6 @@ async def handle_event(self, event): for host in children: # whitelisting / blacklisting based on resolved hosts if rdtype in ("A", "AAAA", "CNAME"): - event.resolved_hosts.add(host) # having a CNAME to an in-scope resource doesn't make you in-scope if not event_whitelisted and rdtype != "CNAME": with suppress(ValidationError): @@ -87,6 +90,7 @@ async def handle_event(self, event): # CNAME to a blacklisted resources, means you're blacklisted with suppress(ValidationError): if self.scan.blacklisted(host): + dns_tags.add("blacklisted") event_blacklisted = True break @@ -98,8 +102,32 @@ async def handle_event(self, event): except ValueError: continue + # store results in cache self._event_cache[event_host_hash] = dns_tags, dns_children, event_whitelisted, event_blacklisted + # abort if the event resolves to something blacklisted + if event_blacklisted: + event.add_tag("blacklisted") + return False, f"blacklisted DNS record" + + # set resolved_hosts attribute + for rdtype, children in dns_children.items(): + for host in children: + event.resolved_hosts.add(host) + + # set dns_children attribute + event.dns_children = dns_children + + # if the event resolves to an in-scope IP, set its scope distance to 0 + if event_whitelisted: + self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") + event.scope_distance = 0 + + # check for wildcards, only if the event resolves to something isn't an IP + if (not event_is_ip) and (dns_children): + if event.scope_distance <= self.scan.scope_search_distance: + await self.handle_wildcard_event(event) + # kill runaway DNS chains dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) if dns_resolve_distance >= self.helpers.dns.max_dns_resolve_distance: @@ -108,31 +136,20 @@ async def handle_event(self, event): ) dns_children = {} + # if the event is a DNS_NAME or IP, tag with "a-record", "ptr-record", etc. if event.type in ("DNS_NAME", "IP_ADDRESS"): - event.dns_children = dns_children for tag in dns_tags: event.add_tag(tag) - if event_blacklisted: - event.add_tag("blacklisted") - reason = "event host" - if event_blacklisted: - reason = "DNS associations" - self.debug(f"Omitting due to blacklisted {reason}: {event}") - - if event_whitelisted: - self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") - event.scope_distance = 0 - - # DNS_NAME --> DNS_NAME_UNRESOLVED + # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: event.type = "DNS_NAME_UNRESOLVED" - - # check for wildcards - if event.scope_distance <= self.scan.scope_search_distance: - if not "unresolved" in event.tags: - if not self.helpers.is_ip_type(event.host): - await self.helpers.dns.handle_wildcard_event(event) + else: + # otherwise, check for wildcards + if event.scope_distance <= self.scan.scope_search_distance: + if not "unresolved" in event.tags: + if not self.helpers.is_ip_type(event.host): + await self.helpers.dns.handle_wildcard_event(event) # speculate DNS_NAMES and IP_ADDRESSes from other event types source_event = event @@ -149,12 +166,12 @@ async def handle_event(self, event): source_event.add_tag("target") self.scan.manager.queue_event(source_event) - ### Emit DNS children ### + # emit DNS children if emit_children: in_dns_scope = -1 < event.scope_distance < self.scope_distance_modifier dns_child_events = [] - if event.dns_children: - for rdtype, records in event.dns_children.items(): + if dns_children: + for rdtype, records in dns_children.items(): module = self.scan._make_dummy_module_dns(rdtype) module._priority = 4 for record in records: @@ -163,7 +180,6 @@ async def handle_event(self, event): # if it's a hostname and it's only one hop away, mark it as affiliate if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: child_event.add_tag("affiliate") - host_hash = hash(str(child_event.host)) if in_dns_scope or self.preset.in_scope(child_event): dns_child_events.append(child_event) except ValidationError as e: @@ -173,3 +189,42 @@ async def handle_event(self, event): for child_event in dns_child_events: self.debug(f"Queueing DNS child for {event}: {child_event}") self.scan.manager.queue_event(child_event) + + async def handle_wildcard_event(self, event): + self.debug(f"Entering handle_wildcard_event({event}, children={event.dns_children})") + try: + event_host = str(event.host) + # check if the dns name itself is a wildcard entry + wildcard_rdtypes = await self.helpers.is_wildcard(event_host) + for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): + wildcard_tag = "error" + if is_wildcard == True: + event.add_tag("wildcard") + wildcard_tag = "wildcard" + event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") + + # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) + if wildcard_rdtypes: + # these are the rdtypes that successfully resolve + resolved_rdtypes = set([c.upper() for c in event.dns_children]) + # these are the rdtypes that have wildcards + wildcard_rdtypes_set = set(wildcard_rdtypes) + # consider the event a full wildcard if all its records are wildcards + event_is_wildcard = False + if resolved_rdtypes: + event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) + + if event_is_wildcard: + if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): + wildcard_parent = self.helpers.parent_domain(event_host) + for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): + if _is_wildcard: + wildcard_parent = _parent_domain + break + wildcard_data = f"_wildcard.{wildcard_parent}" + if wildcard_data != event.data: + self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') + event.data = wildcard_data + + finally: + self.debug(f"Finished handle_wildcard_event({event}, children={event.dns_children})") diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 45887a05d..dd964825c 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -152,7 +152,7 @@ def _event_precheck(self, event): if event._dummy: log.warning(f"Cannot emit dummy event: {event}") return False - if event == event.get_source(): + if (not event.type == "SCAN") and (event == event.get_source()): log.debug(f"Skipping event with self as source: {event}") return False if event._graph_important: diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 07beca1f2..f5f528ac3 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -31,7 +31,7 @@ async def test_dns_engine(bbot_scanner): @pytest.mark.asyncio -async def test_dns(bbot_scanner): +async def test_dns_resolution(bbot_scanner): scan = bbot_scanner("1.1.1.1") from bbot.core.helpers.dns.engine import DNSEngine @@ -108,21 +108,22 @@ async def test_dns(bbot_scanner): assert not hash(f"one.one.one.one:A") in dnsengine._dns_cache # Ensure events with hosts have resolved_hosts attribute populated - resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", dummy=True) - resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", dummy=True) - assert resolved_hosts_event1.host not in scan.helpers.dns._event_cache - assert resolved_hosts_event2.host not in scan.helpers.dns._event_cache - event_tags1, event_whitelisted1, event_blacklisted1, children1 = await scan.helpers.resolve_event( - resolved_hosts_event1 - ) - assert resolved_hosts_event1.host in scan.helpers.dns._event_cache - assert resolved_hosts_event2.host in scan.helpers.dns._event_cache - event_tags2, event_whitelisted2, event_blacklisted2, children2 = await scan.helpers.resolve_event( - resolved_hosts_event2 - ) - assert "1.1.1.1" in [str(x) for x in children1["A"]] - assert "1.1.1.1" in [str(x) for x in children2["A"]] - assert set(children1.keys()) == set(children2.keys()) + await scan._prep() + resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", source=scan.root_event) + resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", source=scan.root_event) + dnsresolve = scan.modules["dnsresolve"] + assert hash(resolved_hosts_event1.host) not in dnsresolve._event_cache + assert hash(resolved_hosts_event2.host) not in dnsresolve._event_cache + await dnsresolve.handle_event(resolved_hosts_event1) + assert hash(resolved_hosts_event1.host) in dnsresolve._event_cache + assert hash(resolved_hosts_event2.host) in dnsresolve._event_cache + await dnsresolve.handle_event(resolved_hosts_event2) + assert "1.1.1.1" in resolved_hosts_event2.resolved_hosts + assert "1.1.1.1" in resolved_hosts_event2.dns_children["A"] + assert resolved_hosts_event1.resolved_hosts == resolved_hosts_event2.resolved_hosts + assert resolved_hosts_event1.dns_children == resolved_hosts_event2.dns_children + assert "a-record" in resolved_hosts_event1.tags + assert not "a-record" in resolved_hosts_event2.tags scan2 = bbot_scanner("evilcorp.com", config={"dns_resolution": True}) await scan2.helpers.dns._mock_dns( @@ -178,12 +179,11 @@ async def test_wildcards(bbot_scanner): wildcard_event3 = scan.make_event("github.io", "DNS_NAME", dummy=True) # event resolution - event_tags1, event_whitelisted1, event_blacklisted1, children1 = await scan.helpers.resolve_event(wildcard_event1) - event_tags2, event_whitelisted2, event_blacklisted2, children2 = await scan.helpers.resolve_event(wildcard_event2) - event_tags3, event_whitelisted3, event_blacklisted3, children3 = await scan.helpers.resolve_event(wildcard_event3) - await helpers.handle_wildcard_event(wildcard_event1, children1) - await helpers.handle_wildcard_event(wildcard_event2, children2) - await helpers.handle_wildcard_event(wildcard_event3, children3) + await scan._prep() + dnsresolve = scan.modules["dnsresolve"] + await dnsresolve.handle_event(wildcard_event1) + await dnsresolve.handle_event(wildcard_event2) + await dnsresolve.handle_event(wildcard_event3) assert "wildcard" in wildcard_event1.tags assert "a-wildcard" in wildcard_event1.tags assert "srv-wildcard" not in wildcard_event1.tags @@ -192,7 +192,43 @@ async def test_wildcards(bbot_scanner): assert "srv-wildcard" not in wildcard_event2.tags assert wildcard_event1.data == "_wildcard.github.io" assert wildcard_event2.data == "_wildcard.github.io" - # TODO: re-enable this? - # assert "wildcard-domain" in wildcard_event3.tags - # assert "a-wildcard-domain" in wildcard_event3.tags - # assert "srv-wildcard-domain" not in wildcard_event3.tags + assert wildcard_event3.data == "github.io" + + from bbot.scanner import Scanner + + # test with full scan + scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dnsresolve": True}) + events = [e async for e in scan2.async_start()] + assert len(events) == 2 + assert 1 == len([e for e in events if e.type == "SCAN"]) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "_wildcard.github.io" + and all( + t in e.tags + for t in ( + "a-record", + "target", + "aaaa-wildcard", + "resolved", + "in-scope", + "subdomain", + "aaaa-record", + "wildcard", + "a-wildcard", + ) + ) + ] + ) + + # test with full scan (wildcard detection disabled for domain) + scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dns_wildcard_ignore": ["github.io"]}) + events = [e async for e in scan2.async_start()] + assert len(events) == 2 + for e in events: + log.critical(e) + # assert 1 == len([e for e in events if e.type == "SCAN"]) + # assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" and all(t in e.tags for t in ('a-record', 'target', 'resolved', 'in-scope', 'subdomain', 'aaaa-record')) and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))]) From 1ee6dc01d3bba7bc8cf85ed6eb07627f53d9fad1 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 10 Apr 2024 10:57:30 -0400 Subject: [PATCH 123/171] flaked --- examples/discord_bot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/discord_bot.py b/examples/discord_bot.py index 61e8bc2b0..f435b0301 100644 --- a/examples/discord_bot.py +++ b/examples/discord_bot.py @@ -1,4 +1,3 @@ -import asyncio import discord from discord.ext import commands From 34b5d3e5a6353ee47d399fcab2badd16989b3270 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 10 Apr 2024 23:37:31 -0400 Subject: [PATCH 124/171] more work on hooks --- bbot/cli.py | 4 +- bbot/modules/base.py | 72 +- bbot/modules/internal/cloudcheck.py | 34 + bbot/modules/internal/dnsresolve.py | 12 +- bbot/scanner/manager.py | 616 +++++------------- bbot/scanner/scanner.py | 190 +++++- .../test_manager_scope_accuracy.py | 4 +- 7 files changed, 397 insertions(+), 535 deletions(-) create mode 100644 bbot/modules/internal/cloudcheck.py diff --git a/bbot/cli.py b/bbot/cli.py index 910b078d6..8e308d6f8 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -184,12 +184,12 @@ def handle_keyboard_input(keyboard_input): module = kill_match.group("module") if module in scan.modules: log.hugewarning(f'Killing module: "{module}"') - scan.manager.kill_module(module, message="killed by user") + scan.kill_module(module, message="killed by user") else: log.warning(f'Invalid module: "{module}"') else: scan.preset.core.logger.toggle_log_level(logger=log) - scan.manager.modules_status(_log=True) + scan.modules_status(_log=True) reader = asyncio.StreamReader() protocol = asyncio.StreamReaderProtocol(reader) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 52d52d656..dbf3f4ce4 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -640,6 +640,8 @@ async def _worker(self): def max_scope_distance(self): if self.in_scope_only or self.target_only: return 0 + if self.scope_distance_modifier is None: + return 999 return max(0, self.scan.scope_search_distance + self.scope_distance_modifier) def _event_precheck(self, event): @@ -775,7 +777,7 @@ async def _cleanup(self): async with self.scan._acatch(context), self._task_counter.count(context): await self.helpers.execute_sync_or_async(callback) - async def queue_event(self, event, precheck=True): + async def queue_event(self, event): """ Asynchronously queues an incoming event to the module's event queue for further processing. @@ -798,9 +800,7 @@ async def queue_event(self, event, precheck=True): if self.incoming_event_queue is False: self.debug(f"Not in an acceptable state to queue incoming event") return - acceptable, reason = True, "precheck was skipped" - if precheck: - acceptable, reason = self._event_precheck(event) + acceptable, reason = self._event_precheck(event) if not acceptable: if reason and reason != "its type is not in watched_events": self.debug(f"Not queueing {event} because {reason}") @@ -812,7 +812,7 @@ async def queue_event(self, event, precheck=True): async with self._event_received: self._event_received.notify() if event.type != "FINISHED": - self.scan.manager._new_activity = True + self.scan._new_activity = True except AttributeError: self.debug(f"Not in an acceptable state to queue incoming event") @@ -1407,23 +1407,18 @@ class HookModule(BaseModule): suppress_dupes = False _hook = True - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._first = False - async def _worker(self): async with self.scan._acatch(context=self._worker, unhandled_is_critical=True): try: while not self.scan.stopping and not self.errored: - try: if self.incoming_event_queue is not False: incoming = await self.get_incoming_event() try: - event, _kwargs = incoming + event, kwargs = incoming except ValueError: event = incoming - _kwargs = {} + kwargs = {} else: self.debug(f"Event queue is in bad state") break @@ -1453,25 +1448,25 @@ async def _worker(self): # whether to pass the event on to the rest of the scan # defaults to true, unless handle_event returns False - pass_on_event = True - pass_on_event_reason = "" + forward_event = True + forward_event_reason = "" if acceptable: - context = f"{self.name}.handle_event({event})" + context = f"{self.name}.handle_event({event, kwargs})" self.scan.stats.event_consumed(event, self) self.debug(f"Hooking {event}") async with self.scan._acatch(context), self._task_counter.count(context): - pass_on_event = await self.handle_event(event) + forward_event = await self.handle_event(event, kwargs) with suppress(ValueError, TypeError): - pass_on_event, pass_on_event_reason = pass_on_event + forward_event, forward_event_reason = forward_event self.debug(f"Finished hooking {event}") - if pass_on_event is False: - self.debug(f"Not passing on {event} because {pass_on_event_reason}") - return + if forward_event is False: + self.debug(f"Not forwarding {event} because {forward_event_reason}") + continue - await self.outgoing_event_queue.put((event, _kwargs)) + await self.forward_event(event, kwargs) except asyncio.CancelledError: self.log.trace("Worker cancelled") @@ -1479,18 +1474,33 @@ async def _worker(self): self.log.trace(f"Worker stopped") async def get_incoming_event(self): - try: - return self.incoming_event_queue.get_nowait() - except asyncio.queues.QueueEmpty: - if self._first: - return self.scan.manager.get_event_from_modules() - raise + """ + Get an event from this module's incoming event queue + """ + return await self.incoming_event_queue.get() - async def queue_event(self, event, precheck=False): + async def forward_event(self, event, kwargs): + """ + Used for forwarding the event on to the next hook module + """ + await self.outgoing_event_queue.put((event, kwargs)) + + async def queue_outgoing_event(self, event, **kwargs): + """ + Used by emit_event() to raise new events to the scan + """ + # if this was a normal module, we'd put it in the outgoing queue + # but because it's a hook module, we need to queue it with the first hook module + await self.scan.ingress_module.queue_event(event, kwargs) + + async def queue_event(self, event, kwargs=None): + """ + Put an event in this module's incoming event queue + """ + if kwargs is None: + kwargs = {} try: - self.incoming_event_queue.put_nowait(event) - if event.type != "FINISHED": - self.scan.manager._new_activity = True + self.incoming_event_queue.put_nowait((event, kwargs)) except AttributeError: self.debug(f"Not in an acceptable state to queue incoming event") diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py new file mode 100644 index 000000000..85dca28aa --- /dev/null +++ b/bbot/modules/internal/cloudcheck.py @@ -0,0 +1,34 @@ +from bbot.modules.base import HookModule + + +class cloudcheck(HookModule): + watched_events = ["*"] + scope_distance_modifier = 1 + _priority = 3 + + async def filter_event(self, event): + if (not event.host) or (event.type in ("IP_RANGE",)): + return False, "event does not have host attribute" + return True + + async def handle_event(self, event, kwargs): + + # skip if we're in tests + if self.helpers.in_tests: + return + + # cloud tagging by main host + await self.scan.helpers.cloud.tag_event(event) + + # cloud tagging by resolved hosts + to_check = set() + if event.type == "IP_ADDRESS": + to_check.add(event.host) + for rdtype, hosts in event.dns_children.items(): + if rdtype in ("A", "AAAA"): + for host in hosts: + to_check.add(host) + for host in to_check: + provider, provider_type, subnet = self.helpers.cloudcheck(host) + if provider: + event.add_tag(f"{provider_type}-{provider}") diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 9ff52e15e..e1771382d 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -33,7 +33,7 @@ async def filter_event(self, event): return False, "event does not have host attribute" return True - async def handle_event(self, event): + async def handle_event(self, event, kwargs): dns_tags = set() dns_children = dict() event_whitelisted = False @@ -144,12 +144,6 @@ async def handle_event(self, event): # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: event.type = "DNS_NAME_UNRESOLVED" - else: - # otherwise, check for wildcards - if event.scope_distance <= self.scan.scope_search_distance: - if not "unresolved" in event.tags: - if not self.helpers.is_ip_type(event.host): - await self.helpers.dns.handle_wildcard_event(event) # speculate DNS_NAMES and IP_ADDRESSes from other event types source_event = event @@ -164,7 +158,7 @@ async def handle_event(self, event): source_event.scope_distance = event.scope_distance if "target" in event.tags: source_event.add_tag("target") - self.scan.manager.queue_event(source_event) + await self.emit_event(source_event) # emit DNS children if emit_children: @@ -188,7 +182,7 @@ async def handle_event(self, event): ) for child_event in dns_child_events: self.debug(f"Queueing DNS child for {event}: {child_event}") - self.scan.manager.queue_event(child_event) + await self.emit_event(child_event) async def handle_wildcard_event(self, event): self.debug(f"Entering handle_wildcard_event({event}, children={event.dns_children})") diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index dd964825c..267175b21 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -1,84 +1,35 @@ import asyncio import logging -import traceback from contextlib import suppress -from ..errors import ValidationError -from ..core.helpers.async_helpers import TaskCounter, ShuffleQueue +from bbot.modules.base import HookModule log = logging.getLogger("bbot.scanner.manager") -class ScanManager: +class ScanIngress(HookModule): """ - Manages the modules, event queues, and overall event flow during a scan. - - Simultaneously serves as a policeman, judge, jury, and executioner for events. - It is responsible for managing the incoming event queue and distributing events to modules. - - Attributes: - scan (Scan): Reference to the Scan object that instantiated the ScanManager. - incoming_event_queue (ShuffleQueue): Queue storing incoming events for processing. - events_distributed (set): Set tracking globally unique events. - events_accepted (set): Set tracking events accepted by individual modules. - dns_resolution (bool): Flag to enable or disable DNS resolution. - _task_counter (TaskCounter): Counter for ongoing tasks. - _new_activity (bool): Flag indicating new activity. - _modules_by_priority (dict): Modules sorted by their priorities. - _incoming_queues (list): List of incoming event queues from each module. - _module_priority_weights (list): Weight values for each module based on priority. + This is always the first hook module in the chain, responsible for basic scope checks """ - def __init__(self, scan): - """ - Initializes the ScanManager object, setting up essential attributes for scan management. - - Args: - scan (Scan): Reference to the Scan object that instantiated the ScanManager. - """ + watched_events = ["*"] + # accept all events regardless of scope distance + scope_distance_modifier = None + _name = "_scan_ingress" - self.scan = scan - self.preset = scan.preset + @property + def priority(self): + # we are the highest priority + return -99 - self.incoming_event_queue = ShuffleQueue() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._module_priority_weights = None + self._non_hook_modules = None # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules) self.incoming_dup_tracker = set() - # track outgoing duplicates (for `accept_dupes` attribute of modules) - self.outgoing_dup_tracker = set() - self.dns_resolution = self.scan.config.get("dns_resolution", False) - self._task_counter = TaskCounter() - self._new_activity = True - self._modules_by_priority = None - self._hook_modules = None - self._non_hook_modules = None - self._incoming_queues = None - self._module_priority_weights = None - async def _worker_loop(self): - try: - while not self.scan.stopped: - try: - async with self._task_counter.count("get_event_from_modules()"): - # if we have hooks set up, we always get events from the last (lowest priority) hook module. - if self.hook_modules: - last_hook_module = self.hook_modules[-1] - event, kwargs = last_hook_module.outgoing_event_queue.get_nowait() - else: - # otherwise, we go through all the modules - event, kwargs = self.get_event_from_modules() - except asyncio.queues.QueueEmpty: - await asyncio.sleep(0.1) - continue - async with self._task_counter.count(f"emit_event({event})"): - emit_event_task = asyncio.create_task( - self.emit_event(event, **kwargs), name=f"emit_event({event})" - ) - await emit_event_task - - except Exception: - log.critical(traceback.format_exc()) - - async def init_events(self): + async def init_events(self, events): """ Initializes events by seeding the scanner with target events and distributing them for further processing. @@ -86,11 +37,8 @@ async def init_events(self): - This method populates the event queue with initial target events. - It also marks the Scan object as finished with initialization by setting `_finished_init` to True. """ - - context = f"manager.init_events()" - async with self.scan._acatch(context), self._task_counter.count(context): - - sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) + async with self.scan._acatch(self.init_events), self._task_counter.count(self.init_events): + sorted_events = sorted(events, key=lambda e: len(e.data)) for event in [self.scan.root_event] + sorted_events: event._dummy = False event.scope_distance = 0 @@ -100,146 +48,67 @@ async def init_events(self): event.source = self.scan.root_event if event.module is None: event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") - self.scan.verbose(f"Target: {event}") - if self.hook_modules: - first_hook_module = self.hook_modules[0] - await first_hook_module.queue_event(event) - else: - self.queue_event(event) + self.verbose(f"Target: {event}") + await self.queue_event(event, {}) await asyncio.sleep(0.1) self.scan._finished_init = True - async def emit_event(self, event, *args, **kwargs): - """ - TODO: Register + kill duplicate events immediately? - bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) - bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) - bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) - """ - callbacks = ["abort_if", "on_success_callback"] - callbacks_requested = any([kwargs.get(k, None) is not None for k in callbacks]) - # "quick" queues the event immediately - # This is used by speculate - quick_kwarg = kwargs.pop("quick", False) - quick_event = getattr(event, "quick_emit", False) - quick = (quick_kwarg or quick_event) and not callbacks_requested - - # skip event if it fails precheck - acceptable = self._event_precheck(event) - if not acceptable: - return - - log.debug(f'Module "{event.module}" raised {event}') - - if quick: - log.debug(f"Quick-emitting {event}") - for kwarg in callbacks: - kwargs.pop(kwarg, None) - async with self.scan._acatch(context=self.distribute_event): - await self.distribute_event(event) - else: - async with self.scan._acatch(context=self._emit_event): - await self._emit_event( - event, - *args, - **kwargs, - ) - - def _event_precheck(self, event): - """ - Check an event to see if we can skip it to save on performance - """ + async def handle_event(self, event, kwargs): + # don't accept dummy events if event._dummy: - log.warning(f"Cannot emit dummy event: {event}") - return False + return False, "cannot emit dummy event" + + # don't accept events with self as source if (not event.type == "SCAN") and (event == event.get_source()): - log.debug(f"Skipping event with self as source: {event}") - return False - if event._graph_important: - return True - if self.is_incoming_duplicate(event, add=True): - log.debug(f"Skipping event because it was already emitted by its module: {event}") - return False - return True + return False, "event's source is itself" - async def _emit_event(self, event, **kwargs): - """ - Handles the emission, tagging, and distribution of a events during a scan. + # don't accept duplicates + if (not event._graph_important) and self.is_incoming_duplicate(event, add=True): + return False, "event was already emitted by its module" - A lot of really important stuff happens here. Actually this is probably the most - important method in all of BBOT. It is basically the central intersection that - every event passes through. + # update event's scope distance based on its parent + event.scope_distance = event.source.scope_distance + 1 - It exists in a delicate balance. Close to half of my debugging time has been spent - in this function. I have slain many dragons here and there may still be more yet to slay. + # blacklist rejections + event_blacklisted = self.scan.blacklisted(event) + if event_blacklisted or "blacklisted" in event.tags: + return False, f"Omitting blacklisted event: {event}" - Tread carefully, friend. -TheTechromancer + # Scope shepherding + # here is where we make sure in-scope events are set to their proper scope distance + event_whitelisted = self.scan.whitelisted(event) + if event.host and event_whitelisted: + log.debug(f"Making {event} in-scope because it matches the scan target") + event.scope_distance = 0 - Notes: - - Central function for decision-making in BBOT. - - Conducts DNS resolution, tagging, and scope calculations. - - Checks against whitelists and blacklists. - - Calls custom callbacks. - - Handles DNS wildcard events. - - Decides on event acceptance and distribution. - - Parameters: - event (Event): The event object to be emitted. - **kwargs: Arbitrary keyword arguments (e.g., `on_success_callback`, `abort_if`). - - Side Effects: - - Event tagging. - - Populating DNS data. - - Emitting new events. - - Queueing events for further processing. - - Adjusting event scopes. - - Running callbacks. - - Updating scan statistics. - """ - log.debug(f"Emitting {event}") - try: - on_success_callback = kwargs.pop("on_success_callback", None) - abort_if = kwargs.pop("abort_if", None) - - # blacklist rejections - event_blacklisted = self.scan.blacklisted(event) - if event_blacklisted or "blacklisted" in event.tags: - log.debug(f"Omitting blacklisted event: {event}") - return - - # Scope shepherding - # here is where we make sure in-scope events are set to their proper scope distance - event_whitelisted = self.scan.whitelisted(event) - if event.host and event_whitelisted: - log.debug(f"Making {event} in-scope because it matches the scan target") - event.scope_distance = 0 + # nerf event's priority if it's not in scope + event.module_priority += event.scope_distance + + @property + def non_hook_modules(self): + if self._non_hook_modules is None: + self._non_hook_modules = [m for m in self.scan.modules.values() if not m._hook] + return self._non_hook_modules - # now that the event is properly tagged, we can finally make decisions about it - abort_result = False - if callable(abort_if): - async with self.scan._acatch(context=abort_if): - abort_result = await self.scan.helpers.execute_sync_or_async(abort_if, event) - msg = f"{event.module}: not raising event {event} due to custom criteria in abort_if()" - with suppress(ValueError, TypeError): - abort_result, reason = abort_result - msg += f": {reason}" - if abort_result: - log.verbose(msg) - return - - # run success callback before distributing event (so it can add tags, etc.) - if callable(on_success_callback): - async with self.scan._acatch(context=on_success_callback): - await self.scan.helpers.execute_sync_or_async(on_success_callback, event) - - await self.distribute_event(event) - - except ValidationError as e: - log.warning(f"Event validation failed with kwargs={kwargs}: {e}") - log.trace(traceback.format_exc()) - - finally: - log.debug(f"{event.module}.emit_event() finished for {event}") + @property + def incoming_queues(self): + return [self.incoming_event_queue] + [m.outgoing_event_queue for m in self.non_hook_modules] + + @property + def module_priority_weights(self): + if not self._module_priority_weights: + # we subtract from six because lower priorities == higher weights + priorities = [5] + [6 - m.priority for m in self.non_hook_modules] + self._module_priority_weights = priorities + return self._module_priority_weights + + async def get_incoming_event(self): + for q in self.helpers.weighted_shuffle(self.incoming_queues, self.module_priority_weights): + try: + return q.get_nowait() + except (asyncio.queues.QueueEmpty, AttributeError): + continue + raise asyncio.queues.QueueEmpty() def is_incoming_duplicate(self, event, add=False): """ @@ -258,6 +127,89 @@ def is_incoming_duplicate(self, event, add=False): return True return False + +class ScanEgress(HookModule): + """ + This is always the last hook module in the chain, responsible for executing and acting on the + `abort_if` and `on_success_callback` functions. + """ + + watched_events = ["*"] + # accept all events regardless of scope distance + scope_distance_modifier = None + _name = "_scan_egress" + + @property + def priority(self): + # we are the lowest priority + return 99 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # track outgoing duplicates (for `accept_dupes` attribute of modules) + self.outgoing_dup_tracker = set() + + async def handle_event(self, event, kwargs): + abort_if = kwargs.pop("abort_if", None) + on_success_callback = kwargs.pop("on_success_callback", None) + + # make event internal if it's above our configured report distance + event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance + event_will_be_output = event.always_emit or event_in_report_distance + if not event_will_be_output: + log.debug( + f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" + ) + event.internal = True + + # if we discovered something interesting from an internal event, + # make sure we preserve its chain of parents + source = event.source + if source.internal and ((not event.internal) or event._graph_important): + source_in_report_distance = source.scope_distance <= self.scan.scope_report_distance + if source_in_report_distance: + source.internal = False + if not source._graph_important: + source._graph_important = True + log.debug(f"Re-queuing internal event {source} with parent {event}") + self.queue_event(source) + + abort_result = False + if callable(abort_if): + async with self.scan._acatch(context=abort_if): + abort_result = await self.scan.helpers.execute_sync_or_async(abort_if, event) + msg = f"{event.module}: not raising event {event} due to custom criteria in abort_if()" + with suppress(ValueError, TypeError): + abort_result, reason = abort_result + msg += f": {reason}" + if abort_result: + return False, msg + + # run success callback before distributing event (so it can add tags, etc.) + if callable(on_success_callback): + async with self.scan._acatch(context=on_success_callback): + await self.scan.helpers.execute_sync_or_async(on_success_callback, event) + + async def forward_event(self, event, kwargs): + """ + Queue event with modules + """ + is_outgoing_duplicate = self.is_outgoing_duplicate(event) + if is_outgoing_duplicate: + self.verbose(f"{event.module}: Duplicate event: {event}") + # absorb event into the word cloud if it's in scope + if not is_outgoing_duplicate and -1 < event.scope_distance < 1: + self.scan.word_cloud.absorb_event(event) + + for mod in self.scan.modules.values(): + # don't distribute events to hook modules + if mod._hook: + continue + acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes + graph_important = mod._is_graph_important(event) + if acceptable_dup or graph_important: + await mod.queue_event(event) + def is_outgoing_duplicate(self, event, add=False): """ Calculate whether an event is a duplicate in the context of the whole scan, @@ -270,269 +222,3 @@ def is_outgoing_duplicate(self, event, add=False): if add: self.outgoing_dup_tracker.add(event_hash) return is_dup - - async def distribute_event(self, event): - """ - Queue event with modules - """ - async with self.scan._acatch(context=self.distribute_event): - # make event internal if it's above our configured report distance - event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance - event_will_be_output = event.always_emit or event_in_report_distance - if not event_will_be_output: - log.debug( - f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" - ) - event.internal = True - - # if we discovered something interesting from an internal event, - # make sure we preserve its chain of parents - source = event.source - if source.internal and ((not event.internal) or event._graph_important): - source_in_report_distance = source.scope_distance <= self.scan.scope_report_distance - if source_in_report_distance: - source.internal = False - if not source._graph_important: - source._graph_important = True - log.debug(f"Re-queuing internal event {source} with parent {event}") - self.queue_event(source) - - is_outgoing_duplicate = self.is_outgoing_duplicate(event) - if is_outgoing_duplicate: - self.scan.verbose(f"{event.module}: Duplicate event: {event}") - # absorb event into the word cloud if it's in scope - if not is_outgoing_duplicate and -1 < event.scope_distance < 1: - self.scan.word_cloud.absorb_event(event) - for mod in self.scan.modules.values(): - # don't distribute events to hook modules - if mod._hook: - continue - acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes - # graph_important = mod._type == "output" and event._graph_important == True - graph_important = mod._is_graph_important(event) - if acceptable_dup or graph_important: - await mod.queue_event(event) - - def kill_module(self, module_name, message=None): - from signal import SIGINT - - module = self.scan.modules[module_name] - module.set_error_state(message=message, clear_outgoing_queue=True) - for proc in module._proc_tracker: - with suppress(Exception): - proc.send_signal(SIGINT) - self.scan.helpers.cancel_tasks_sync(module._tasks) - - @property - def modules_by_priority(self): - if not self._modules_by_priority: - self._modules_by_priority = sorted(list(self.scan.modules.values()), key=lambda m: m.priority) - return self._modules_by_priority - - @property - def incoming_queues(self): - if not self._incoming_queues: - queues_by_priority = [m.outgoing_event_queue for m in self.modules_by_priority if not m._hook] - self._incoming_queues = [self.incoming_event_queue] + queues_by_priority - return self._incoming_queues - - @property - def incoming_qsize(self): - incoming_events = 0 - for q in self.incoming_queues: - incoming_events += q.qsize() - return incoming_events - - @property - def module_priority_weights(self): - if not self._module_priority_weights: - # we subtract from six because lower priorities == higher weights - priorities = [5] + [6 - m.priority for m in self.modules_by_priority if not m._hook] - self._module_priority_weights = priorities - return self._module_priority_weights - - @property - def hook_modules(self): - if self._hook_modules is None: - self._hook_modules = [m for m in self.modules_by_priority if m._hook] - if self._hook_modules: - self._hook_modules[0]._first = True - return self._hook_modules - - @property - def non_hook_modules(self): - if self._non_hook_modules is None: - self._non_hook_modules = [m for m in self.modules_by_priority if not m._hook] - return self._non_hook_modules - - def get_event_from_modules(self): - for q in self.scan.helpers.weighted_shuffle(self.incoming_queues, self.module_priority_weights): - try: - return q.get_nowait() - except (asyncio.queues.QueueEmpty, AttributeError): - continue - raise asyncio.queues.QueueEmpty() - - @property - def queued_event_types(self): - event_types = {} - for q in self.incoming_queues: - for event, _ in q._queue: - event_type = getattr(event, "type", None) - if event_type is not None: - try: - event_types[event_type] += 1 - except KeyError: - event_types[event_type] = 1 - return event_types - - def queue_event(self, event, **kwargs): - if event: - # nerf event's priority if it's likely not to be in scope - if event.scope_distance > 0: - event_in_scope = self.scan.whitelisted(event) and not self.scan.blacklisted(event) - if not event_in_scope: - event.module_priority += event.scope_distance - # update event's scope distance based on its parent - event.scope_distance = event.source.scope_distance + 1 - self.incoming_event_queue.put_nowait((event, kwargs)) - - @property - def running(self): - active_tasks = self._task_counter.value - incoming_events = self.incoming_qsize - return active_tasks > 0 or incoming_events > 0 - - @property - def modules_finished(self): - finished_modules = [m.finished for m in self.scan.modules.values()] - return all(finished_modules) - - @property - def active(self): - return self.running or not self.modules_finished - - def modules_status(self, _log=False): - finished = True - status = {"modules": {}} - - for m in self.scan.modules.values(): - mod_status = m.status - if mod_status["running"]: - finished = False - status["modules"][m.name] = mod_status - - for mod in self.scan.modules.values(): - if mod.errored and mod.incoming_event_queue not in [None, False]: - with suppress(Exception): - mod.set_error_state() - - status["finished"] = finished - - modules_errored = [m for m, s in status["modules"].items() if s["errored"]] - - max_mem_percent = 90 - mem_status = self.scan.helpers.memory_status() - # abort if we don't have the memory - mem_percent = mem_status.percent - if mem_percent > max_mem_percent: - free_memory = mem_status.available - free_memory_human = self.scan.helpers.bytes_to_human(free_memory) - self.scan.warning(f"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining)") - - if _log: - modules_status = [] - for m, s in status["modules"].items(): - running = s["running"] - incoming = s["events"]["incoming"] - outgoing = s["events"]["outgoing"] - tasks = s["tasks"] - total = sum([incoming, outgoing, tasks]) - if running or total > 0: - modules_status.append((m, running, incoming, outgoing, tasks, total)) - modules_status.sort(key=lambda x: x[-1], reverse=True) - - if modules_status: - modules_status_str = ", ".join([f"{m}({i:,}:{t:,}:{o:,})" for m, r, i, o, t, _ in modules_status]) - self.scan.info( - f"{self.scan.name}: Modules running (incoming:processing:outgoing) {modules_status_str}" - ) - else: - self.scan.info(f"{self.scan.name}: No modules running") - event_type_summary = sorted( - self.scan.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True - ) - if event_type_summary: - self.scan.info( - f'{self.scan.name}: Events produced so far: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}' - ) - else: - self.scan.info(f"{self.scan.name}: No events produced yet") - - if modules_errored: - self.scan.verbose( - f'{self.scan.name}: Modules errored: {len(modules_errored):,} ({", ".join([m for m in modules_errored])})' - ) - - queued_events_by_type = [(k, v) for k, v in self.queued_event_types.items() if v > 0] - if queued_events_by_type: - queued_events_by_type.sort(key=lambda x: x[-1], reverse=True) - queued_events_by_type_str = ", ".join(f"{m}: {t:,}" for m, t in queued_events_by_type) - num_queued_events = sum(v for k, v in queued_events_by_type) - self.scan.info( - f"{self.scan.name}: {num_queued_events:,} events in queue ({queued_events_by_type_str})" - ) - else: - self.scan.info(f"{self.scan.name}: No events in queue") - - if self.scan.log_level <= logging.DEBUG: - # status debugging - scan_active_status = [] - scan_active_status.append(f"scan._finished_init: {self.scan._finished_init}") - scan_active_status.append(f"manager.active: {self.active}") - scan_active_status.append(f" manager.running: {self.running}") - scan_active_status.append(f" manager._task_counter.value: {self._task_counter.value}") - scan_active_status.append(f" manager._task_counter.tasks:") - for task in list(self._task_counter.tasks.values()): - scan_active_status.append(f" - {task}:") - scan_active_status.append( - f" manager.incoming_event_queue.qsize: {self.incoming_event_queue.qsize()}" - ) - scan_active_status.append(f" manager.modules_finished: {self.modules_finished}") - for m in sorted(self.scan.modules.values(), key=lambda m: m.name): - running = m.running - scan_active_status.append(f" {m}.finished: {m.finished}") - scan_active_status.append(f" running: {running}") - if running: - scan_active_status.append(f" tasks:") - for task in list(m._task_counter.tasks.values()): - scan_active_status.append(f" - {task}:") - scan_active_status.append(f" incoming_queue_size: {m.num_incoming_events}") - scan_active_status.append(f" outgoing_queue_size: {m.outgoing_event_queue.qsize()}") - for line in scan_active_status: - self.scan.debug(line) - - # log module memory usage - module_memory_usage = [] - for module in self.scan.modules.values(): - memory_usage = module.memory_usage - module_memory_usage.append((module.name, memory_usage)) - module_memory_usage.sort(key=lambda x: x[-1], reverse=True) - self.scan.debug(f"MODULE MEMORY USAGE:") - for module_name, usage in module_memory_usage: - self.scan.debug(f" - {module_name}: {self.scan.helpers.bytes_to_human(usage)}") - - # Uncomment these lines to enable debugging of event queues - - # queued_events = self.incoming_event_queue.events - # if queued_events: - # queued_events_str = ", ".join(str(e) for e in queued_events) - # self.scan.verbose(f"Queued events: {queued_events_str}") - # queued_events_by_module = [(k, v) for k, v in self.incoming_event_queue.modules.items() if v > 0] - # queued_events_by_module.sort(key=lambda x: x[-1], reverse=True) - # queued_events_by_module_str = ", ".join(f"{m}: {t:,}" for m, t in queued_events_by_module) - # self.scan.verbose(f"{self.scan.name}: Queued events by module: {queued_events_by_module_str}") - - status.update({"modules_errored": len(modules_errored)}) - - return status diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 42a199d57..13b3bfde8 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -17,9 +17,9 @@ from .preset import Preset from .stats import ScanStats -from .manager import ScanManager from .dispatcher import Dispatcher from bbot.core.event import make_event +from .manager import ScanIngress, ScanEgress from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.names_generator import random_name from bbot.core.helpers.async_helpers import async_to_sync_gen @@ -74,7 +74,6 @@ class Scanner: helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. (alias to `self.preset.helpers`). output_dir (pathlib.Path): Output directory for scan (alias to `self.preset.output_dir`). name (str): Name of scan (alias to `self.preset.scan_name`). - manager (ScanManager): Coordinates and monitors the flow of events between modules during a scan. dispatcher (Dispatcher): Triggers certain events when the scan `status` changes. modules (dict): Holds all loaded modules in this format: `{"module_name": Module()}`. stats (ScanStats): Holds high-level scan statistics such as how many events have been produced and consumed by each module. @@ -177,7 +176,6 @@ def __init__( self.dispatcher = dispatcher self.dispatcher.set_scan(self) - self.manager = ScanManager(self) self.stats = ScanStats(self) # scope distance @@ -200,6 +198,7 @@ def __init__( self._prepped = False self._finished_init = False + self._new_activity = False self._cleanedup = False self.__loop = None @@ -308,17 +307,12 @@ async def async_start(self): await self.dispatcher.on_start(self) - # start manager worker loops - self._manager_worker_loop_tasks = [ - asyncio.create_task(self.manager._worker_loop()) for _ in range(self.max_workers) - ] - self.status = "RUNNING" self._start_modules() self.verbose(f"{len(self.modules):,} modules started") # distribute seed events - self.init_events_task = asyncio.create_task(self.manager.init_events()) + self.init_events_task = asyncio.create_task(self.ingress_module.init_events(self.target.events)) # main scan loop while 1: @@ -334,7 +328,7 @@ async def async_start(self): yield e # break if initialization finished and the scan is no longer active - if self._finished_init and not self.manager.active: + if self._finished_init and self.modules_finished: new_activity = await self.finish() if not new_activity: break @@ -385,16 +379,6 @@ async def async_start(self): def _start_modules(self): self.verbose(f"Starting module worker loops") - - # hook modules get sewn together like human centipede - if len(self.manager.hook_modules) > 1: - for i, hook_module in enumerate(self.manager.hook_modules[:-1]): - next_hook_module = self.manager.hook_modules[i + 1] - self.debug( - f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" - ) - hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue - for module in self.modules.values(): module.start() @@ -520,9 +504,164 @@ async def load_modules(self): f"Loaded {len(loaded_output_modules):,}/{len(self.preset.output_modules):,} output modules, ({','.join(loaded_output_modules)})" ) - self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "_priority", 0))) + # builtin hook modules + self.ingress_module = ScanIngress(self) + self.egress_module = ScanEgress(self) + self.modules[self.ingress_module.name] = self.ingress_module + self.modules[self.egress_module.name] = self.egress_module + + # sort modules by priority + self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "priority", 3))) + + self.critical(list(self.modules)) + + # hook modules get sewn together like human centipede + self.hook_modules = [m for m in self.modules.values() if m._hook] + for i, hook_module in enumerate(self.hook_modules[:-1]): + next_hook_module = self.hook_modules[i + 1] + self.debug( + f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" + ) + hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue + self._modules_loaded = True + @property + def modules_finished(self): + finished_modules = [m.finished for m in self.modules.values()] + return all(finished_modules) + + def kill_module(self, module_name, message=None): + from signal import SIGINT + + module = self.modules[module_name] + module.set_error_state(message=message, clear_outgoing_queue=True) + for proc in module._proc_tracker: + with contextlib.suppress(Exception): + proc.send_signal(SIGINT) + self.helpers.cancel_tasks_sync(module._tasks) + + @property + def queued_event_types(self): + event_types = {} + queues = set() + + for module in self.modules.values(): + queues.add(module.incoming_event_queue) + queues.add(module.outgoing_event_queue) + + for q in queues: + for event, _ in q._queue: + event_type = getattr(event, "type", None) + if event_type is not None: + try: + event_types[event_type] += 1 + except KeyError: + event_types[event_type] = 1 + + return event_types + + def modules_status(self, _log=False): + finished = True + status = {"modules": {}} + + sorted_modules = [] + for module_name, module in self.modules.items(): + # if module_name.startswith("_"): + # continue + sorted_modules.append(module) + mod_status = module.status + if mod_status["running"]: + finished = False + status["modules"][module_name] = mod_status + + # sort modules by name + sorted_modules.sort(key=lambda m: m.name) + + status["finished"] = finished + + modules_errored = [m for m, s in status["modules"].items() if s["errored"]] + + max_mem_percent = 90 + mem_status = self.helpers.memory_status() + # abort if we don't have the memory + mem_percent = mem_status.percent + if mem_percent > max_mem_percent: + free_memory = mem_status.available + free_memory_human = self.helpers.bytes_to_human(free_memory) + self.warning(f"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining)") + + if _log: + modules_status = [] + for m, s in status["modules"].items(): + running = s["running"] + incoming = s["events"]["incoming"] + outgoing = s["events"]["outgoing"] + tasks = s["tasks"] + total = sum([incoming, outgoing, tasks]) + if running or total > 0: + modules_status.append((m, running, incoming, outgoing, tasks, total)) + modules_status.sort(key=lambda x: x[-1], reverse=True) + + if modules_status: + modules_status_str = ", ".join([f"{m}({i:,}:{t:,}:{o:,})" for m, r, i, o, t, _ in modules_status]) + self.info(f"{self.name}: Modules running (incoming:processing:outgoing) {modules_status_str}") + else: + self.info(f"{self.name}: No modules running") + event_type_summary = sorted(self.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True) + if event_type_summary: + self.info( + f'{self.name}: Events produced so far: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}' + ) + else: + self.info(f"{self.name}: No events produced yet") + + if modules_errored: + self.verbose( + f'{self.name}: Modules errored: {len(modules_errored):,} ({", ".join([m for m in modules_errored])})' + ) + + queued_events_by_type = [(k, v) for k, v in self.queued_event_types.items() if v > 0] + if queued_events_by_type: + queued_events_by_type.sort(key=lambda x: x[-1], reverse=True) + queued_events_by_type_str = ", ".join(f"{m}: {t:,}" for m, t in queued_events_by_type) + num_queued_events = sum(v for k, v in queued_events_by_type) + self.info(f"{self.name}: {num_queued_events:,} events in queue ({queued_events_by_type_str})") + else: + self.info(f"{self.name}: No events in queue") + + if self.log_level <= logging.DEBUG: + # status debugging + scan_active_status = [] + scan_active_status.append(f"scan._finished_init: {self._finished_init}") + scan_active_status.append(f"scan.modules_finished: {self.modules_finished}") + for m in sorted_modules: + running = m.running + scan_active_status.append(f" {m}.finished: {m.finished}") + scan_active_status.append(f" running: {running}") + if running: + scan_active_status.append(f" tasks:") + for task in list(m._task_counter.tasks.values()): + scan_active_status.append(f" - {task}:") + scan_active_status.append(f" incoming_queue_size: {m.num_incoming_events}") + scan_active_status.append(f" outgoing_queue_size: {m.outgoing_event_queue.qsize()}") + for line in scan_active_status: + self.debug(line) + + # log module memory usage + module_memory_usage = [] + for module in sorted_modules: + memory_usage = module.memory_usage + module_memory_usage.append((module.name, memory_usage)) + module_memory_usage.sort(key=lambda x: x[-1], reverse=True) + self.debug(f"MODULE MEMORY USAGE:") + for module_name, usage in module_memory_usage: + self.debug(f" - {module_name}: {self.helpers.bytes_to_human(usage)}") + + status.update({"modules_errored": len(modules_errored)}) + + return status + def stop(self): """Stops the in-progress scan and performs necessary cleanup. @@ -555,8 +694,8 @@ async def finish(self): This method alters the scan's status to "FINISHING" if new activity is detected. """ # if new events were generated since last time we were here - if self.manager._new_activity: - self.manager._new_activity = False + if self._new_activity: + self._new_activity = False self.status = "FINISHING" # Trigger .finished() on every module and start over log.info("Finishing scan") @@ -587,9 +726,6 @@ def _drain_queues(self): while 1: if module.outgoing_event_queue: module.outgoing_event_queue.get_nowait() - with contextlib.suppress(asyncio.queues.QueueEmpty): - while 1: - self.manager.incoming_event_queue.get_nowait() self.debug("Finished draining queues") def _cancel_tasks(self): @@ -997,7 +1133,7 @@ async def _status_ticker(self, interval=15): async with self._acatch(): while 1: await asyncio.sleep(interval) - self.manager.modules_status(_log=True) + self.modules_status(_log=True) @contextlib.asynccontextmanager async def _acatch(self, context="scan", finally_callback=None, unhandled_is_critical=False): diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 2d4fece4f..da1b87037 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -177,7 +177,9 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) - assert len(all_events) == 9 + # assert len(all_events) == 9 + for e in all_events: + log.critical(e) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1]) assert 2 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1]) From eae2dad3d12aed5cf684bc5f00d2a142ea6383a3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 10 Apr 2024 23:38:54 -0400 Subject: [PATCH 125/171] remove debugging statement --- bbot/scanner/scanner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 13b3bfde8..a11ba49fb 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -513,8 +513,6 @@ async def load_modules(self): # sort modules by priority self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "priority", 3))) - self.critical(list(self.modules)) - # hook modules get sewn together like human centipede self.hook_modules = [m for m in self.modules.values() if m._hook] for i, hook_module in enumerate(self.hook_modules[:-1]): From 1165cfd23298a312bbd95d5ca582dbb4ec3b836f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 11 Apr 2024 11:36:24 -0400 Subject: [PATCH 126/171] remove return statement in scope accuracy tests --- bbot/test/test_step_1/test_manager_scope_accuracy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index b0f957ee3..ca14fbf98 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -340,8 +340,6 @@ def custom_setup(scan): assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) - return - assert len(all_events) == 14 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) From 17598f700735674005c6915e793fdf2fc7c3e99f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 14 Apr 2024 17:27:21 -0400 Subject: [PATCH 127/171] dns tests passing --- bbot/modules/base.py | 2 +- bbot/modules/internal/dnsresolve.py | 34 ++++++++++------ bbot/modules/sslcert.py | 7 +++- bbot/scanner/manager.py | 5 ++- bbot/scanner/scanner.py | 39 ++++++++++--------- .../test_manager_scope_accuracy.py | 13 +++---- 6 files changed, 59 insertions(+), 41 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index dbf3f4ce4..0f8867201 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -565,7 +565,7 @@ async def _setup(self): self.set_error_state(f"Unexpected error during module setup: {e}", critical=True) msg = f"{e}" self.trace() - return self.name, status, str(msg) + return self, status, str(msg) async def _worker(self): """ diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index e1771382d..1056d9343 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -12,6 +12,7 @@ class dnsresolve(HookModule): watched_events = ["*"] _priority = 1 _max_event_handlers = 25 + scope_distance_modifier = None async def setup(self): self.dns_resolution = self.scan.config.get("dns_resolution", False) @@ -25,7 +26,7 @@ async def setup(self): return True @property - def scope_distance_modifier(self): + def _dns_search_distance(self): return max(self.scope_search_distance, self.scope_dns_search_distance) async def filter_event(self, event): @@ -43,8 +44,11 @@ async def handle_event(self, event, kwargs): event_host_hash = hash(str(event.host)) event_is_ip = self.helpers.is_ip(event.host) + # whether we've reached the max scope distance for dns + within_dns_search_distance = event.scope_distance < self._dns_search_distance + # only emit DNS children if we haven't seen this host before - emit_children = event_host_hash not in self._event_cache + emit_children = self.dns_resolution and event_host_hash not in self._event_cache # we do DNS resolution inside a lock to make sure we don't duplicate work # once the resolution happens, it will be cached so it doesn't need to happen again @@ -53,8 +57,16 @@ async def handle_event(self, event, kwargs): # try to get from cache dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] except KeyError: + if event_is_ip: + rdtypes_to_resolve = ["PTR"] + else: + if self.dns_resolution and within_dns_search_distance: + rdtypes_to_resolve = all_rdtypes + else: + rdtypes_to_resolve = ["A", "AAAA", "CNAME"] + # if missing from cache, do DNS resolution - queries = [(event_host, rdtype) for rdtype in all_rdtypes] + queries = [(event_host, rdtype) for rdtype in rdtypes_to_resolve] error_rdtypes = [] async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): if errors: @@ -70,11 +82,10 @@ async def handle_event(self, event, kwargs): if rdtype not in dns_children: dns_tags.add(f"{rdtype.lower()}-error") - if not event_is_ip: - if dns_children: - dns_tags.add("resolved") - else: - dns_tags.add("unresolved") + if dns_children: + dns_tags.add("resolved") + elif not event_is_ip: + dns_tags.add("unresolved") for rdtype, children in dns_children.items(): if event_blacklisted: @@ -108,7 +119,7 @@ async def handle_event(self, event, kwargs): # abort if the event resolves to something blacklisted if event_blacklisted: event.add_tag("blacklisted") - return False, f"blacklisted DNS record" + return False, f"it has a blacklisted DNS record" # set resolved_hosts attribute for rdtype, children in dns_children.items(): @@ -152,7 +163,8 @@ async def handle_event(self, event, kwargs): and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") ): - source_event = self.make_event(event.host, "DNS_NAME", source=event) + source_module = self.scan._make_dummy_module("host", _type="internal") + source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event) # only emit the event if it's not already in the parent chain if source_event is not None and source_event not in event.get_sources(): source_event.scope_distance = event.scope_distance @@ -162,7 +174,7 @@ async def handle_event(self, event, kwargs): # emit DNS children if emit_children: - in_dns_scope = -1 < event.scope_distance < self.scope_distance_modifier + in_dns_scope = -1 < event.scope_distance < self._dns_search_distance dns_child_events = [] if dns_children: for rdtype, records in dns_children.items(): diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 357826920..c6fec1ea9 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -78,8 +78,13 @@ async def handle_event(self, event): self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]") try: ssl_event = self.make_event(event_data, event_type, source=event, raise_error=True) + source_event = ssl_event.get_source() + if source_event.scope_distance == 0: + tags = ["affiliate"] + else: + tags = None if ssl_event: - await self.emit_event(ssl_event, on_success_callback=self.on_success_callback) + await self.emit_event(ssl_event, tags=tags) except ValidationError as e: self.hugeinfo(f'Malformed {event_type} "{event_data}" at {event.data}') self.debug(f"Invalid data at {host}:{port}: {e}") diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 267175b21..f390bb65b 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -118,7 +118,8 @@ def is_incoming_duplicate(self, event, add=False): try: event_hash = event.module._outgoing_dedup_hash(event) except AttributeError: - event_hash = hash((event, str(getattr(event, "module", "")))) + module_name = str(getattr(event, "module", "")) + event_hash = hash((event, module_name)) is_dup = event_hash in self.incoming_dup_tracker if add: self.incoming_dup_tracker.add(event_hash) @@ -172,7 +173,7 @@ async def handle_event(self, event, kwargs): if not source._graph_important: source._graph_important = True log.debug(f"Re-queuing internal event {source} with parent {event}") - self.queue_event(source) + await self.emit_event(source) abort_result = False if callable(abort_if): diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a11ba49fb..f64af069d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -252,6 +252,15 @@ async def _prep(self): # run each module's .setup() method succeeded, hard_failed, soft_failed = await self.setup_modules() + # hook modules get sewn together like human centipede + self.hook_modules = [m for m in self.modules.values() if m._hook] + for i, hook_module in enumerate(self.hook_modules[:-1]): + next_hook_module = self.hook_modules[i + 1] + self.debug( + f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" + ) + hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue + # abort if there are no output modules num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) if num_output_modules < 1: @@ -408,19 +417,20 @@ async def setup_modules(self, remove_failed=True): soft_failed = [] async for task in self.helpers.as_completed([m._setup() for m in self.modules.values()]): - module_name, status, msg = await task + module, status, msg = await task if status == True: - self.debug(f"Setup succeeded for {module_name} ({msg})") - succeeded.append(module_name) + self.debug(f"Setup succeeded for {module.name} ({msg})") + succeeded.append(module.name) elif status == False: - self.warning(f"Setup hard-failed for {module_name}: {msg}") - self.modules[module_name].set_error_state() - hard_failed.append(module_name) + self.warning(f"Setup hard-failed for {module.name}: {msg}") + self.modules[module.name].set_error_state() + hard_failed.append(module.name) else: - self.info(f"Setup soft-failed for {module_name}: {msg}") - soft_failed.append(module_name) - if not status and remove_failed: - self.modules.pop(module_name) + self.info(f"Setup soft-failed for {module.name}: {msg}") + soft_failed.append(module.name) + if (not status) and (module._hook or remove_failed): + # if a hook module fails setup, we always remove it + self.modules.pop(module.name) return succeeded, hard_failed, soft_failed @@ -513,15 +523,6 @@ async def load_modules(self): # sort modules by priority self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "priority", 3))) - # hook modules get sewn together like human centipede - self.hook_modules = [m for m in self.modules.values() if m._hook] - for i, hook_module in enumerate(self.hook_modules[:-1]): - next_hook_module = self.hook_modules[i + 1] - self.debug( - f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" - ) - hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue - self._modules_loaded = True @property diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 8f429c32e..9ee2c72e5 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -111,8 +111,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) scan.modules["dummy_module_nodupes"] = dummy_module_nodupes scan.modules["dummy_graph_output_module"] = dummy_graph_output_module scan.modules["dummy_graph_batch_output_module"] = dummy_graph_batch_output_module - if _dns_mock: - await scan.helpers.dns._mock_dns(_dns_mock) + await scan.helpers.dns._mock_dns(_dns_mock) if scan_callback is not None: scan_callback(scan) return ( @@ -177,9 +176,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) - # assert len(all_events) == 9 - for e in all_events: - log.critical(e) + assert len(all_events) == 9 assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1]) assert 2 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1]) @@ -327,6 +324,7 @@ def custom_setup(scan): "modules": {"speculate": {"ports": "8888"}}, "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], }, + _dns_mock={}, ) assert len(events) == 6 @@ -567,6 +565,7 @@ def custom_setup(scan): modules=["httpx"], output_modules=["python"], _config={ + "dns_resolution": True, "scope_search_distance": 0, "scope_dns_search_distance": 2, "scope_report_distance": 0, @@ -743,7 +742,7 @@ def custom_setup(scan): assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == False and e.scope_distance == 1 and str(e.module) == "sslcert"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "www.bbottest.notreal:9999"]) - assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "bbottest.notreal"]) + assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "bbottest.notreal"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) # sslcert with out-of-scope chain @@ -821,7 +820,7 @@ async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog): # the hostname is in-scope, but its IP is blacklisted, therefore we shouldn't see it assert not any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-prod.test.notreal:8888/"]) - assert 'Omitting due to blacklisted DNS associations: URL_UNVERIFIED("http://www-prod.test.notreal:8888/"' in caplog.text + assert 'Not forwarding DNS_NAME("www-prod.test.notreal", module=excavate' in caplog.text and 'because it has a blacklisted DNS record' in caplog.text @pytest.mark.asyncio From dfd7b8858dc4888ce4fbee86ed0d140b598fda09 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 00:28:19 -0400 Subject: [PATCH 128/171] cloudcheck defragmentation --- bbot/core/helpers/cloud.py | 92 ---------------- bbot/core/helpers/helper.py | 8 +- bbot/modules/internal/cloud.py | 73 +++++++++++++ bbot/modules/internal/cloudcheck.py | 34 ------ .../internal/{dnsresolve.py => dns.py} | 13 ++- bbot/modules/internal/excavate.py | 8 -- bbot/modules/internal/speculate.py | 8 -- bbot/modules/templates/bucket.py | 2 +- bbot/test/test_step_1/test_cloud_helpers.py | 86 --------------- .../module_tests/test_module_cloud.py | 102 ++++++++++++++++++ 10 files changed, 188 insertions(+), 238 deletions(-) delete mode 100644 bbot/core/helpers/cloud.py create mode 100644 bbot/modules/internal/cloud.py delete mode 100644 bbot/modules/internal/cloudcheck.py rename bbot/modules/internal/{dnsresolve.py => dns.py} (96%) delete mode 100644 bbot/test/test_step_1/test_cloud_helpers.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_cloud.py diff --git a/bbot/core/helpers/cloud.py b/bbot/core/helpers/cloud.py deleted file mode 100644 index 7f1e19b69..000000000 --- a/bbot/core/helpers/cloud.py +++ /dev/null @@ -1,92 +0,0 @@ -import asyncio -import logging - -from cloudcheck import cloud_providers - -log = logging.getLogger("bbot.helpers.cloud") - - -class CloudHelper: - def __init__(self, parent_helper): - self.parent_helper = parent_helper - self.providers = cloud_providers - self._updated = False - self._update_lock = asyncio.Lock() - - def excavate(self, event, s): - """ - Extract buckets, etc. from strings such as an HTTP responses - """ - for provider in self: - provider_name = provider.name.lower() - base_kwargs = {"source": event, "tags": [f"cloud-{provider_name}"], "_provider": provider_name} - for event_type, sigs in provider.signatures.items(): - found = set() - for sig in sigs: - for match in sig.findall(s): - kwargs = dict(base_kwargs) - kwargs["event_type"] = event_type - if not match in found: - found.add(match) - if event_type == "STORAGE_BUCKET": - yield self.emit_bucket(match, **kwargs) - else: - yield kwargs - - def speculate(self, event): - """ - Look for DNS_NAMEs that are buckets or other cloud resources - """ - for provider in self: - provider_name = provider.name.lower() - base_kwargs = dict( - source=event, tags=[f"{provider.provider_type}-{provider_name}"], _provider=provider_name - ) - if event.type.startswith("DNS_NAME"): - for event_type, sigs in provider.signatures.items(): - found = set() - for sig in sigs: - match = sig.match(event.data) - if match: - kwargs = dict(base_kwargs) - kwargs["event_type"] = event_type - if not event.data in found: - found.add(event.data) - if event_type == "STORAGE_BUCKET": - yield self.emit_bucket(match.groups(), **kwargs) - else: - yield kwargs - - def emit_bucket(self, match, **kwargs): - bucket_name, bucket_domain = match - kwargs["data"] = {"name": bucket_name, "url": f"https://{bucket_name}.{bucket_domain}"} - return kwargs - - async def tag_event(self, event): - """ - Tags an event according to cloud provider - """ - async with self._update_lock: - if not self._updated: - await self.providers.update() - self._updated = True - - if event.host: - for host in [event.host] + list(event.resolved_hosts): - provider_name, provider_type, source = self.providers.check(host) - if provider_name is not None: - provider = self.providers.providers[provider_name.lower()] - event.add_tag(f"{provider_type}-{provider_name.lower()}") - # if its host directly matches this cloud provider's domains - if not self.parent_helper.is_ip(host): - # tag as buckets, etc. - for event_type, sigs in provider.signatures.items(): - for sig in sigs: - if sig.match(host): - event.add_tag(f"{provider_type}-{event_type}") - - def __getitem__(self, item): - return self.providers.providers[item.lower()] - - def __iter__(self): - yield from self.providers diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 8cc923fde..d2bb4bb19 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -1,18 +1,17 @@ import os import logging from pathlib import Path +from cloudcheck import cloud_providers from . import misc from .dns import DNSHelper from .web import WebHelper from .diff import HttpCompare -from .cloud import CloudHelper from .wordcloud import WordCloud from .interactsh import Interactsh from ...scanner.target import Target from .depsinstaller import DepsInstaller - log = logging.getLogger("bbot.core.helpers") @@ -66,15 +65,14 @@ def __init__(self, preset): self.mkdir(self.tools_dir) self.mkdir(self.lib_dir) + self.cloud = cloud_providers + self.dns = DNSHelper(self) self.web = WebHelper(self) self.depsinstaller = DepsInstaller(self) self.word_cloud = WordCloud(self) self.dummy_modules = {} - # cloud helpers - self.cloud = CloudHelper(self) - def interactsh(self, *args, **kwargs): return Interactsh(self, *args, **kwargs) diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py new file mode 100644 index 000000000..3ed5f7f50 --- /dev/null +++ b/bbot/modules/internal/cloud.py @@ -0,0 +1,73 @@ +from bbot.modules.base import HookModule + + +class cloud(HookModule): + watched_events = ["*"] + meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"} + scope_distance_modifier = 1 + _priority = 3 + + async def setup(self): + self.dummy_modules = {} + for provider_name in self.helpers.cloud.providers: + self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan") + return True + + async def filter_event(self, event): + if (not event.host) or (event.type in ("IP_RANGE",)): + return False, "event does not have host attribute" + return True + + async def handle_event(self, event, kwargs): + # cloud tagging by hosts + hosts_to_check = set(str(s) for s in event.resolved_hosts) + hosts_to_check.add(str(event.host)) + for host in hosts_to_check: + provider, provider_type, subnet = self.helpers.cloudcheck(host) + if provider: + event.add_tag(f"{provider_type}-{provider}") + + found = set() + # look for cloud assets in hosts, http responses + # loop through each provider + for provider in self.helpers.cloud.providers.values(): + provider_name = provider.name.lower() + base_kwargs = dict( + source=event, tags=[f"{provider.provider_type}-{provider_name}"], _provider=provider_name + ) + # loop through the provider's regex signatures, if any + for event_type, sigs in provider.signatures.items(): + if event_type != "STORAGE_BUCKET": + raise ValueError(f'Unknown cloudcheck event type "{event_type}"') + base_kwargs["event_type"] = event_type + for sig in sigs: + matches = [] + if event.type == "HTTP_RESPONSE": + matches = sig.findall(event.data.get("body", "")) + elif event.type.startswith("DNS_NAME"): + for host in hosts_to_check: + matches.append(sig.match(host)) + for match in matches: + if not match: + continue + if not event.data in found: + found.add(event.data) + if event_type == "STORAGE_BUCKET": + _kwargs = dict(base_kwargs) + event_type_tag = f"cloud-{event_type}" + _kwargs["tags"].append(event_type_tag) + if event.type.startswith("DNS_NAME"): + event.add_tag(event_type_tag) + bucket_name, bucket_domain = match.groups() + _kwargs["data"] = { + "name": bucket_name, + "url": f"https://{bucket_name}.{bucket_domain}", + } + await self.emit_event(**_kwargs) + + async def emit_event(self, *args, **kwargs): + provider_name = kwargs.pop("_provider") + dummy_module = self.dummy_modules[provider_name] + event = dummy_module.make_event(*args, **kwargs) + if event: + await super().emit_event(event) diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py deleted file mode 100644 index 85dca28aa..000000000 --- a/bbot/modules/internal/cloudcheck.py +++ /dev/null @@ -1,34 +0,0 @@ -from bbot.modules.base import HookModule - - -class cloudcheck(HookModule): - watched_events = ["*"] - scope_distance_modifier = 1 - _priority = 3 - - async def filter_event(self, event): - if (not event.host) or (event.type in ("IP_RANGE",)): - return False, "event does not have host attribute" - return True - - async def handle_event(self, event, kwargs): - - # skip if we're in tests - if self.helpers.in_tests: - return - - # cloud tagging by main host - await self.scan.helpers.cloud.tag_event(event) - - # cloud tagging by resolved hosts - to_check = set() - if event.type == "IP_ADDRESS": - to_check.add(event.host) - for rdtype, hosts in event.dns_children.items(): - if rdtype in ("A", "AAAA"): - for host in hosts: - to_check.add(host) - for host in to_check: - provider, provider_type, subnet = self.helpers.cloudcheck(host) - if provider: - event.add_tag(f"{provider_type}-{provider}") diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dns.py similarity index 96% rename from bbot/modules/internal/dnsresolve.py rename to bbot/modules/internal/dns.py index 1056d9343..173d9129a 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dns.py @@ -8,14 +8,18 @@ from bbot.core.helpers.async_helpers import NamedLock -class dnsresolve(HookModule): +class DNS(HookModule): watched_events = ["*"] _priority = 1 _max_event_handlers = 25 scope_distance_modifier = None async def setup(self): - self.dns_resolution = self.scan.config.get("dns_resolution", False) + self.dns_resolution = True + # you can disable DNS resolution with either the "dns" or "dns_resolution" config options + for key in ("dns", "dns_resolution"): + if self.scan.config.get(key, None) is False: + self.dns_resolution = False self.scope_search_distance = max(0, int(self.scan.config.get("scope_search_distance", 0))) self.scope_dns_search_distance = max(0, int(self.scan.config.get("scope_dns_search_distance", 1))) @@ -123,8 +127,9 @@ async def handle_event(self, event, kwargs): # set resolved_hosts attribute for rdtype, children in dns_children.items(): - for host in children: - event.resolved_hosts.add(host) + if rdtype in ("A", "AAAA", "CNAME"): + for host in children: + event.resolved_hosts.add(host) # set dns_children attribute event.dns_children = dns_children diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 0ec91ce67..1af70b051 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -388,14 +388,6 @@ async def handle_event(self, event): body = self.helpers.recursive_decode(event.data.get("body", "")) - # Cloud extractors - for cloud_kwargs in self.helpers.cloud.excavate(event, body): - module = None - provider = cloud_kwargs.pop("_provider", "") - if provider: - module = self.scan._make_dummy_module(f"{provider}_cloud") - await self.emit_event(module=module, **cloud_kwargs) - await self.search( body, [ diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index e6d778a05..f983af6bf 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -142,14 +142,6 @@ async def handle_event(self, event): quick=True, ) - # storage buckets etc. - for cloud_kwargs in self.helpers.cloud.speculate(event): - module = None - provider = cloud_kwargs.pop("_provider", "") - if provider: - module = self.scan._make_dummy_module(provider) - await self.emit_event(module=module, **cloud_kwargs) - # ORG_STUB from TLD, SOCIAL, AZURE_TENANT org_stubs = set() if event.type == "DNS_NAME" and event.scope_distance == 0: diff --git a/bbot/modules/templates/bucket.py b/bbot/modules/templates/bucket.py index 095da6b70..04597faa2 100644 --- a/bbot/modules/templates/bucket.py +++ b/bbot/modules/templates/bucket.py @@ -19,7 +19,7 @@ class bucket_template(BaseModule): async def setup(self): self.buckets_tried = set() - self.cloud_helper = self.helpers.cloud[self.cloud_helper_name] + self.cloud_helper = self.helpers.cloud.providers[self.cloud_helper_name] self.permutations = self.config.get("permutations", False) return True diff --git a/bbot/test/test_step_1/test_cloud_helpers.py b/bbot/test/test_step_1/test_cloud_helpers.py deleted file mode 100644 index adfc290ca..000000000 --- a/bbot/test/test_step_1/test_cloud_helpers.py +++ /dev/null @@ -1,86 +0,0 @@ -from ..bbot_fixtures import * # noqa: F401 - - -@pytest.mark.asyncio -async def test_cloud_helpers(bbot_scanner): - scan1 = bbot_scanner("127.0.0.1") - - provider_names = ("amazon", "google", "azure", "digitalocean", "oracle", "akamai", "cloudflare", "github") - for provider_name in provider_names: - assert provider_name in scan1.helpers.cloud.providers.providers - - for p in scan1.helpers.cloud.providers.providers.values(): - print(f"{p.name}: {p.domains} / {p.ranges}") - amazon_ranges = list(scan1.helpers.cloud["amazon"].ranges) - assert amazon_ranges - amazon_range = next(iter(amazon_ranges)) - amazon_address = amazon_range.broadcast_address - - ip_event = scan1.make_event(amazon_address, source=scan1.root_event) - aws_event1 = scan1.make_event("amazonaws.com", source=scan1.root_event) - aws_event2 = scan1.make_event("asdf.amazonaws.com", source=scan1.root_event) - aws_event3 = scan1.make_event("asdfamazonaws.com", source=scan1.root_event) - aws_event4 = scan1.make_event("test.asdf.aws", source=scan1.root_event) - - other_event1 = scan1.make_event("cname.evilcorp.com", source=scan1.root_event) - other_event2 = scan1.make_event("cname2.evilcorp.com", source=scan1.root_event) - other_event3 = scan1.make_event("cname3.evilcorp.com", source=scan1.root_event) - other_event2._resolved_hosts = {amazon_address} - other_event3._resolved_hosts = {"asdf.amazonaws.com"} - - for event in (ip_event, aws_event1, aws_event2, aws_event4, other_event2, other_event3): - await scan1.helpers.cloud.tag_event(event) - assert "cloud-amazon" in event.tags, f"{event} was not properly cloud-tagged" - - for event in (aws_event3, other_event1): - await scan1.helpers.cloud.tag_event(event) - assert "cloud-amazon" not in event.tags, f"{event} was improperly cloud-tagged" - assert not any( - t for t in event.tags if t.startswith("cloud-") or t.startswith("cdn-") - ), f"{event} was improperly cloud-tagged" - - google_event1 = scan1.make_event("asdf.googleapis.com", source=scan1.root_event) - google_event2 = scan1.make_event("asdf.google", source=scan1.root_event) - google_event3 = scan1.make_event("asdf.evilcorp.com", source=scan1.root_event) - google_event3._resolved_hosts = {"asdf.storage.googleapis.com"} - - for event in (google_event1, google_event2, google_event3): - await scan1.helpers.cloud.tag_event(event) - assert "cloud-google" in event.tags, f"{event} was not properly cloud-tagged" - assert "cloud-storage-bucket" in google_event3.tags - - -@pytest.mark.asyncio -async def test_cloud_helpers_excavate(bbot_scanner, bbot_httpserver): - url = bbot_httpserver.url_for("/test_cloud_helpers_excavate") - bbot_httpserver.expect_request(uri="/test_cloud_helpers_excavate").respond_with_data( - "" - ) - scan1 = bbot_scanner(url, modules=["httpx"], config={"excavate": True}) - events = [e async for e in scan1.async_start()] - assert 1 == len( - [ - e - for e in events - if e.type == "STORAGE_BUCKET" - and e.data["name"] == "asdf" - and "cloud-amazon" in e.tags - and "cloud-storage-bucket" in e.tags - ] - ) - - -@pytest.mark.asyncio -async def test_cloud_helpers_speculate(bbot_scanner): - scan1 = bbot_scanner("asdf.s3.amazonaws.com", config={"speculate": True}) - events = [e async for e in scan1.async_start()] - assert 1 == len( - [ - e - for e in events - if e.type == "STORAGE_BUCKET" - and e.data["name"] == "asdf" - and "cloud-amazon" in e.tags - and "cloud-storage-bucket" in e.tags - ] - ) diff --git a/bbot/test/test_step_2/module_tests/test_module_cloud.py b/bbot/test/test_step_2/module_tests/test_module_cloud.py new file mode 100644 index 000000000..89942ed06 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_cloud.py @@ -0,0 +1,102 @@ +from .base import ModuleTestBase + + +class TestCloud(ModuleTestBase): + targets = ["www.azure.com"] + + async def setup_after_prep(self, module_test): + scan = module_test.scan + module = module_test.module + providers = scan.helpers.cloud.providers + # make sure we have all the providers + provider_names = ( + "amazon", + "google", + "azure", + "digitalocean", + "oracle", + "akamai", + "cloudflare", + "github", + "zoho", + "fastly", + ) + for provider_name in provider_names: + assert provider_name in providers + + amazon_ranges = list(providers["amazon"].ranges) + assert amazon_ranges + amazon_range = next(iter(amazon_ranges)) + amazon_address = amazon_range.broadcast_address + + ip_event = scan.make_event(amazon_address, source=scan.root_event) + aws_event1 = scan.make_event("amazonaws.com", source=scan.root_event) + aws_event2 = scan.make_event("asdf.amazonaws.com", source=scan.root_event) + aws_event3 = scan.make_event("asdfamazonaws.com", source=scan.root_event) + aws_event4 = scan.make_event("test.asdf.aws", source=scan.root_event) + + other_event1 = scan.make_event("cname.evilcorp.com", source=scan.root_event) + other_event2 = scan.make_event("cname2.evilcorp.com", source=scan.root_event) + other_event3 = scan.make_event("cname3.evilcorp.com", source=scan.root_event) + other_event2._resolved_hosts = {amazon_address} + other_event3._resolved_hosts = {"asdf.amazonaws.com"} + + for event in (ip_event, aws_event1, aws_event2, aws_event4, other_event2, other_event3): + await module.handle_event(event, {}) + assert "cloud-amazon" in event.tags, f"{event} was not properly cloud-tagged" + + for event in (aws_event3, other_event1): + await module.handle_event(event, {}) + assert "cloud-amazon" not in event.tags, f"{event} was improperly cloud-tagged" + assert not any( + t for t in event.tags if t.startswith("cloud-") or t.startswith("cdn-") + ), f"{event} was improperly cloud-tagged" + + google_event1 = scan.make_event("asdf.googleapis.com", source=scan.root_event) + google_event2 = scan.make_event("asdf.google", source=scan.root_event) + google_event3 = scan.make_event("asdf.evilcorp.com", source=scan.root_event) + google_event3._resolved_hosts = {"asdf.storage.googleapis.com"} + + for event in (google_event1, google_event2, google_event3): + await module.handle_event(event, {}) + assert "cloud-google" in event.tags, f"{event} was not properly cloud-tagged" + assert "cloud-storage-bucket" in google_event3.tags + + def check(self, events, module_test): + pass + + +# @pytest.mark.asyncio +# async def test_cloud_helpers_excavate(bbot_scanner, bbot_httpserver): +# url = bbot_httpserver.url_for("/test_cloud_helpers_excavate") +# bbot_httpserver.expect_request(uri="/test_cloud_helpers_excavate").respond_with_data( +# "" +# ) +# scan = bbot_scanner(url, modules=["httpx"], config={"excavate": True}) +# events = [e async for e in scan.async_start()] +# assert 1 == len( +# [ +# e +# for e in events +# if e.type == "STORAGE_BUCKET" +# and e.data["name"] == "asdf" +# and "cloud-amazon" in e.tags +# and "cloud-storage-bucket" in e.tags +# ] +# ) + + +# @pytest.mark.asyncio +# async def test_cloud_helpers_speculate(bbot_scanner): +# scan = bbot_scanner("asdf.s3.amazonaws.com", config={"speculate": True}) +# events = [e async for e in scan.async_start()] +# assert 1 == len( +# [ +# e +# for e in events +# if e.type == "STORAGE_BUCKET" +# and e.data["name"] == "asdf" +# and "cloud-amazon" in e.tags +# and "cloud-storage-bucket" in e.tags +# ] +# ) From 06a4d38e0710a5a5dc5bfce33b6b6aee77604549 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 10:53:37 -0400 Subject: [PATCH 129/171] dns module tests --- bbot/core/event/base.py | 20 ++++- bbot/modules/internal/cloud.py | 26 ++++--- bbot/test/test_step_1/test_events.py | 7 ++ .../module_tests/test_module_cloud.py | 73 ++++++++----------- .../module_tests/test_module_dns.py | 62 ++++++++++++++++ 5 files changed, 133 insertions(+), 55 deletions(-) create mode 100644 bbot/test/test_step_2/module_tests/test_module_dns.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 345e64115..74b476ee6 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -151,6 +151,7 @@ def __init__( self._port = None self.__words = None self._priority = None + self._host_original = None self._module_priority = None self._resolved_hosts = set() self.dns_children = dict() @@ -275,9 +276,24 @@ def host(self): E.g. for IP_ADDRESS, it could be an ipaddress.IPv4Address() or IPv6Address() object """ if self.__host is None: - self.__host = self._host() + self.host = self._host() return self.__host + @host.setter + def host(self, host): + if self._host_original is None: + self._host_original = host + self.__host = host + + @property + def host_original(self): + """ + Original host data, in case it was changed due to a wildcard DNS, etc. + """ + if self._host_original is None: + return self.host + return self._host_original + @property def port(self): self.host @@ -793,7 +809,7 @@ def __init__(self, *args, **kwargs): ip = ipaddress.ip_address(self.data) self.add_tag(f"ipv{ip.version}") if ip.is_private: - self.add_tag("private") + self.add_tag("private-ip") self.dns_resolve_distance = getattr(self.source, "dns_resolve_distance", 0) def sanitize_data(self, data): diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index 3ed5f7f50..bf69baa35 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -21,7 +21,7 @@ async def filter_event(self, event): async def handle_event(self, event, kwargs): # cloud tagging by hosts hosts_to_check = set(str(s) for s in event.resolved_hosts) - hosts_to_check.add(str(event.host)) + hosts_to_check.add(str(event.host_original)) for host in hosts_to_check: provider, provider_type, subnet = self.helpers.cloudcheck(host) if provider: @@ -46,19 +46,21 @@ async def handle_event(self, event, kwargs): matches = sig.findall(event.data.get("body", "")) elif event.type.startswith("DNS_NAME"): for host in hosts_to_check: - matches.append(sig.match(host)) + match = sig.match(host) + if match: + matches.append(match.groups()) for match in matches: - if not match: - continue - if not event.data in found: - found.add(event.data) + if not match in found: + found.add(match) + + _kwargs = dict(base_kwargs) + event_type_tag = f"cloud-{event_type}" + _kwargs["tags"].append(event_type_tag) + if event.type.startswith("DNS_NAME"): + event.add_tag(event_type_tag) + if event_type == "STORAGE_BUCKET": - _kwargs = dict(base_kwargs) - event_type_tag = f"cloud-{event_type}" - _kwargs["tags"].append(event_type_tag) - if event.type.startswith("DNS_NAME"): - event.add_tag(event_type_tag) - bucket_name, bucket_domain = match.groups() + bucket_name, bucket_domain = match _kwargs["data"] = { "name": bucket_name, "url": f"https://{bucket_name}.{bucket_domain}", diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 211ef4f36..541234677 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -444,3 +444,10 @@ async def test_events(events, scan, helpers): event_5 = scan.make_event("127.0.0.5", source=event_4) assert event_5.get_sources() == [event_4, event_3, event_2, event_1, scan.root_event] assert event_5.get_sources(omit=True) == [event_4, event_2, event_1, scan.root_event] + + # test host backup + host_event = scan.make_event("asdf.evilcorp.com", "DNS_NAME", source=scan.root_event) + assert host_event.host_original == "asdf.evilcorp.com" + host_event.host = "_wildcard.evilcorp.com" + assert host_event.host == "_wildcard.evilcorp.com" + assert host_event.host_original == "asdf.evilcorp.com" diff --git a/bbot/test/test_step_2/module_tests/test_module_cloud.py b/bbot/test/test_step_2/module_tests/test_module_cloud.py index 89942ed06..1d4e59283 100644 --- a/bbot/test/test_step_2/module_tests/test_module_cloud.py +++ b/bbot/test/test_step_2/module_tests/test_module_cloud.py @@ -1,12 +1,19 @@ from .base import ModuleTestBase +from bbot.scanner import Scanner + class TestCloud(ModuleTestBase): - targets = ["www.azure.com"] + targets = ["http://127.0.0.1:8888", "asdf2.storage.googleapis.com"] + modules_overrides = ["httpx", "excavate", "cloud"] async def setup_after_prep(self, module_test): - scan = module_test.scan - module = module_test.module + + module_test.set_expect_requests({"uri": "/"}, {"response_data": ""}) + + scan = Scanner(config={"cloud": True}) + await scan._prep() + module = scan.modules["cloud"] providers = scan.helpers.cloud.providers # make sure we have all the providers provider_names = ( @@ -62,41 +69,25 @@ async def setup_after_prep(self, module_test): assert "cloud-google" in event.tags, f"{event} was not properly cloud-tagged" assert "cloud-storage-bucket" in google_event3.tags - def check(self, events, module_test): - pass - - -# @pytest.mark.asyncio -# async def test_cloud_helpers_excavate(bbot_scanner, bbot_httpserver): -# url = bbot_httpserver.url_for("/test_cloud_helpers_excavate") -# bbot_httpserver.expect_request(uri="/test_cloud_helpers_excavate").respond_with_data( -# "" -# ) -# scan = bbot_scanner(url, modules=["httpx"], config={"excavate": True}) -# events = [e async for e in scan.async_start()] -# assert 1 == len( -# [ -# e -# for e in events -# if e.type == "STORAGE_BUCKET" -# and e.data["name"] == "asdf" -# and "cloud-amazon" in e.tags -# and "cloud-storage-bucket" in e.tags -# ] -# ) - - -# @pytest.mark.asyncio -# async def test_cloud_helpers_speculate(bbot_scanner): -# scan = bbot_scanner("asdf.s3.amazonaws.com", config={"speculate": True}) -# events = [e async for e in scan.async_start()] -# assert 1 == len( -# [ -# e -# for e in events -# if e.type == "STORAGE_BUCKET" -# and e.data["name"] == "asdf" -# and "cloud-amazon" in e.tags -# and "cloud-storage-bucket" in e.tags -# ] -# ) + def check(self, module_test, events): + assert 2 == len([e for e in events if e.type == "STORAGE_BUCKET"]) + assert 1 == len( + [ + e + for e in events + if e.type == "STORAGE_BUCKET" + and e.data["name"] == "asdf" + and "cloud-amazon" in e.tags + and "cloud-storage-bucket" in e.tags + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "STORAGE_BUCKET" + and e.data["name"] == "asdf2" + and "cloud-google" in e.tags + and "cloud-storage-bucket" in e.tags + ] + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_dns.py b/bbot/test/test_step_2/module_tests/test_module_dns.py new file mode 100644 index 000000000..0351ac44e --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_dns.py @@ -0,0 +1,62 @@ +from .base import ModuleTestBase + + +class TestDNS(ModuleTestBase): + modules_overrides = ["dns"] + config_overrides = {"dns_resolution": True, "scope_report_distance": 1} + + async def setup_after_prep(self, module_test): + await module_test.mock_dns( + { + "blacklanternsecurity.com": { + "A": ["192.168.0.7"], + "AAAA": ["::1"], + "CNAME": ["www.blacklanternsecurity.com"], + }, + "www.blacklanternsecurity.com": {"A": ["192.168.0.8"]}, + } + ) + + def check(self, module_test, events): + self.log.critical(events) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "blacklanternsecurity.com" + and "a-record" in e.tags + and "aaaa-record" in e.tags + and "cname-record" in e.tags + and "private-ip" in e.tags + and e.scope_distance == 0 + and "192.168.0.7" in e.resolved_hosts + and "::1" in e.resolved_hosts + and "www.blacklanternsecurity.com" in e.resolved_hosts + and e.dns_children + == {"A": {"192.168.0.7"}, "AAAA": {"::1"}, "CNAME": {"www.blacklanternsecurity.com"}} + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "www.blacklanternsecurity.com" + and "a-record" in e.tags + and "private-ip" in e.tags + and e.scope_distance == 0 + and "192.168.0.8" in e.resolved_hosts + and e.dns_children == {"A": {"192.168.0.8"}} + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "IP_ADDRESS" + and e.data == "192.168.0.7" + and "private-ip" in e.tags + and e.scope_distance == 1 + ] + ) From ebb729580bacd6d421dd2e2636f4fae0160d3d03 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 10:56:26 -0400 Subject: [PATCH 130/171] fix dns tests --- bbot/test/test_step_1/test_dns.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index f5f528ac3..7c1fda190 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -111,13 +111,13 @@ async def test_dns_resolution(bbot_scanner): await scan._prep() resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", source=scan.root_event) resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", source=scan.root_event) - dnsresolve = scan.modules["dnsresolve"] + dnsresolve = scan.modules["dns"] assert hash(resolved_hosts_event1.host) not in dnsresolve._event_cache assert hash(resolved_hosts_event2.host) not in dnsresolve._event_cache - await dnsresolve.handle_event(resolved_hosts_event1) + await dnsresolve.handle_event(resolved_hosts_event1, {}) assert hash(resolved_hosts_event1.host) in dnsresolve._event_cache assert hash(resolved_hosts_event2.host) in dnsresolve._event_cache - await dnsresolve.handle_event(resolved_hosts_event2) + await dnsresolve.handle_event(resolved_hosts_event2, {}) assert "1.1.1.1" in resolved_hosts_event2.resolved_hosts assert "1.1.1.1" in resolved_hosts_event2.dns_children["A"] assert resolved_hosts_event1.resolved_hosts == resolved_hosts_event2.resolved_hosts @@ -180,10 +180,10 @@ async def test_wildcards(bbot_scanner): # event resolution await scan._prep() - dnsresolve = scan.modules["dnsresolve"] - await dnsresolve.handle_event(wildcard_event1) - await dnsresolve.handle_event(wildcard_event2) - await dnsresolve.handle_event(wildcard_event3) + dnsresolve = scan.modules["dns"] + await dnsresolve.handle_event(wildcard_event1, {}) + await dnsresolve.handle_event(wildcard_event2, {}) + await dnsresolve.handle_event(wildcard_event3, {}) assert "wildcard" in wildcard_event1.tags assert "a-wildcard" in wildcard_event1.tags assert "srv-wildcard" not in wildcard_event1.tags @@ -197,7 +197,7 @@ async def test_wildcards(bbot_scanner): from bbot.scanner import Scanner # test with full scan - scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dnsresolve": True}) + scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dns_resolution": True}) events = [e async for e in scan2.async_start()] assert len(events) == 2 assert 1 == len([e for e in events if e.type == "SCAN"]) @@ -228,7 +228,5 @@ async def test_wildcards(bbot_scanner): scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dns_wildcard_ignore": ["github.io"]}) events = [e async for e in scan2.async_start()] assert len(events) == 2 - for e in events: - log.critical(e) - # assert 1 == len([e for e in events if e.type == "SCAN"]) - # assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" and all(t in e.tags for t in ('a-record', 'target', 'resolved', 'in-scope', 'subdomain', 'aaaa-record')) and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))]) + assert 1 == len([e for e in events if e.type == "SCAN"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" and all(t in e.tags for t in ('a-record', 'target', 'resolved', 'in-scope', 'subdomain', 'aaaa-record')) and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))]) From c4e1a19ef04a74e7e2b8e6902ba98c2073870aa7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 10:56:36 -0400 Subject: [PATCH 131/171] blacked --- bbot/test/test_step_1/test_dns.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 7c1fda190..0355656b8 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -229,4 +229,13 @@ async def test_wildcards(bbot_scanner): events = [e async for e in scan2.async_start()] assert len(events) == 2 assert 1 == len([e for e in events if e.type == "SCAN"]) - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" and all(t in e.tags for t in ('a-record', 'target', 'resolved', 'in-scope', 'subdomain', 'aaaa-record')) and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))]) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "asdfl.gashdgkjsadgsdf.github.io" + and all(t in e.tags for t in ("a-record", "target", "resolved", "in-scope", "subdomain", "aaaa-record")) + and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard")) + ] + ) From db3961c262045004fe3655705fcc9d35125a2b7f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 11:34:44 -0400 Subject: [PATCH 132/171] fix module tests --- bbot/test/test_step_1/test_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 6ada6e64e..db2a5316d 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -98,17 +98,17 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): monkeypatch.setattr("sys.argv", ["bbot", "-y"]) result = await cli._main() assert result == True - assert "Loaded 3/3 internal modules (aggregate,excavate,speculate)" in caplog.text + assert "Loaded 5/5 internal modules (aggregate,cloud,dns,excavate,speculate)" in caplog.text caplog.clear() monkeypatch.setattr("sys.argv", ["bbot", "-em", "excavate", "speculate", "-y"]) result = await cli._main() assert result == True - assert "Loaded 1/1 internal modules (aggregate)" in caplog.text + assert "Loaded 3/3 internal modules (aggregate,cloud,dns)" in caplog.text caplog.clear() monkeypatch.setattr("sys.argv", ["bbot", "-c", "speculate=false", "-y"]) result = await cli._main() assert result == True - assert "Loaded 2/2 internal modules (aggregate,excavate)" in caplog.text + assert "Loaded 4/4 internal modules (aggregate,cloud,dns,excavate)" in caplog.text # list modules caplog.clear() From b2679d2168b41a06c77fff9141ed10c13d1d8923 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 12:21:47 -0400 Subject: [PATCH 133/171] quick emit revisit --- bbot/scanner/manager.py | 8 ++++++++ bbot/scanner/stats.py | 3 +++ bbot/test/test_step_1/test_modules_basic.py | 6 +++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index f390bb65b..2fd218861 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -84,6 +84,14 @@ async def handle_event(self, event, kwargs): # nerf event's priority if it's not in scope event.module_priority += event.scope_distance + async def forward_event(self, event, kwargs): + # if a module qualifies for "quick-emit", we skip all the intermediate modules like dns and cloud + # and forward it straight to the egress module + if event.quick_emit: + await self.scan.egress_module.queue_event(event, kwargs) + else: + await super().forward_event(event, kwargs) + @property def non_hook_modules(self): if self._non_hook_modules is None: diff --git a/bbot/scanner/stats.py b/bbot/scanner/stats.py index 0c0a4d287..617703d8c 100644 --- a/bbot/scanner/stats.py +++ b/bbot/scanner/stats.py @@ -23,6 +23,9 @@ def event_produced(self, event): module_stat.increment_produced(event) def event_consumed(self, event, module): + # skip ingress/egress modules, etc. + if module.name.startswith("_"): + return module_stat = self.get(module) if module_stat is not None: module_stat.increment_consumed(event) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 7f01428e6..5672e616e 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -352,7 +352,7 @@ async def handle_event(self, event): "ORG_STUB": 1, } - assert set(scan.stats.module_stats) == {"host", "speculate", "python", "dummy", "TARGET"} + assert set(scan.stats.module_stats) == {'speculate', 'host', 'TARGET', 'python', 'dummy', 'cloud', 'dns'} target_stats = scan.stats.module_stats["TARGET"] assert target_stats.produced == {"SCAN": 1, "DNS_NAME": 1} @@ -363,8 +363,8 @@ async def handle_event(self, event): dummy_stats = scan.stats.module_stats["dummy"] assert dummy_stats.produced == {"FINDING": 1, "URL": 1} assert dummy_stats.produced_total == 2 - assert dummy_stats.consumed == {"DNS_NAME": 2, "OPEN_TCP_PORT": 1, "SCAN": 1, "URL": 1, "URL_UNVERIFIED": 1} - assert dummy_stats.consumed_total == 6 + assert dummy_stats.consumed == {"DNS_NAME": 2, "FINDING": 1, "OPEN_TCP_PORT": 1, "SCAN": 1, "URL": 1, "URL_UNVERIFIED": 1} + assert dummy_stats.consumed_total == 7 python_stats = scan.stats.module_stats["python"] assert python_stats.produced == {} From 5746f6e840499f45c57ecfc48888350c93329430 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 12:21:56 -0400 Subject: [PATCH 134/171] blacked --- bbot/test/test_step_1/test_modules_basic.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 5672e616e..5fc187fe3 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -352,7 +352,7 @@ async def handle_event(self, event): "ORG_STUB": 1, } - assert set(scan.stats.module_stats) == {'speculate', 'host', 'TARGET', 'python', 'dummy', 'cloud', 'dns'} + assert set(scan.stats.module_stats) == {"speculate", "host", "TARGET", "python", "dummy", "cloud", "dns"} target_stats = scan.stats.module_stats["TARGET"] assert target_stats.produced == {"SCAN": 1, "DNS_NAME": 1} @@ -363,7 +363,14 @@ async def handle_event(self, event): dummy_stats = scan.stats.module_stats["dummy"] assert dummy_stats.produced == {"FINDING": 1, "URL": 1} assert dummy_stats.produced_total == 2 - assert dummy_stats.consumed == {"DNS_NAME": 2, "FINDING": 1, "OPEN_TCP_PORT": 1, "SCAN": 1, "URL": 1, "URL_UNVERIFIED": 1} + assert dummy_stats.consumed == { + "DNS_NAME": 2, + "FINDING": 1, + "OPEN_TCP_PORT": 1, + "SCAN": 1, + "URL": 1, + "URL_UNVERIFIED": 1, + } assert dummy_stats.consumed_total == 7 python_stats = scan.stats.module_stats["python"] From 3b0efcff5e77c1c6d5e2068059179f0104ebdec0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 12:37:45 -0400 Subject: [PATCH 135/171] fix preset tests --- bbot/test/test_step_1/test_presets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 864f1d0c4..d84244e4f 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -254,7 +254,7 @@ def test_preset_module_resolution(clean_default_config): # make sure we have the expected defaults assert not preset.scan_modules assert set(preset.output_modules) == {"python", "csv", "human", "json"} - assert set(preset.internal_modules) == {"aggregate", "excavate", "speculate"} + assert set(preset.internal_modules) == {"aggregate", "excavate", "speculate", "cloud", "dns"} assert preset.modules == set(preset.output_modules).union(set(preset.internal_modules)) # make sure dependency resolution works as expected From 30f325f6be57baeba8d4d38506c1f645a6991d66 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 13:02:57 -0400 Subject: [PATCH 136/171] fix affiliate tests --- bbot/modules/internal/dns.py | 8 +------- bbot/scanner/scanner.py | 6 +++++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index 173d9129a..12e5608d9 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -48,9 +48,6 @@ async def handle_event(self, event, kwargs): event_host_hash = hash(str(event.host)) event_is_ip = self.helpers.is_ip(event.host) - # whether we've reached the max scope distance for dns - within_dns_search_distance = event.scope_distance < self._dns_search_distance - # only emit DNS children if we haven't seen this host before emit_children = self.dns_resolution and event_host_hash not in self._event_cache @@ -64,10 +61,7 @@ async def handle_event(self, event, kwargs): if event_is_ip: rdtypes_to_resolve = ["PTR"] else: - if self.dns_resolution and within_dns_search_distance: - rdtypes_to_resolve = all_rdtypes - else: - rdtypes_to_resolve = ["A", "AAAA", "CNAME"] + rdtypes_to_resolve = all_rdtypes # if missing from cache, do DNS resolution queries = [(event_host, rdtype) for rdtype in rdtypes_to_resolve] diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index f64af069d..4a8ebf0f6 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -550,7 +550,11 @@ def queued_event_types(self): queues.add(module.outgoing_event_queue) for q in queues: - for event, _ in q._queue: + for item in q._queue: + try: + event, _ = item + except ValueError: + event = item event_type = getattr(event, "type", None) if event_type is not None: try: From b29356b9d795eb439c08c6a53c5f7452b6e029dd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 13:21:50 -0400 Subject: [PATCH 137/171] fix tests --- bbot/core/helpers/dns/dns.py | 4 ++++ bbot/defaults.yml | 6 +++++- bbot/modules/internal/dns.py | 2 ++ bbot/test/test_step_1/test_dns.py | 14 ++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index cc0a1ff4a..9764687bf 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -62,6 +62,7 @@ def __init__(self, parent_helper): self.max_dns_resolve_distance = self.config.get("max_dns_resolve_distance", 5) # wildcard handling + self.wildcard_disable = self.config.get("dns_wildcard_disable", False) self.wildcard_ignore = self.config.get("dns_wildcard_ignore", None) if not self.wildcard_ignore: self.wildcard_ignore = [] @@ -136,6 +137,9 @@ async def is_wildcard_domain(self, domain, log_info=False): return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) def _wildcard_prevalidation(self, host): + if self.wildcard_disable: + return False + host = clean_dns_record(host) # skip check if it's an IP or a plain hostname if is_ip(host) or not "." in host: diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 5b6323ae4..9eefb838b 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -79,11 +79,14 @@ httpx_retries: 1 http_debug: false # Maximum number of HTTP redirects to follow http_max_redirects: 5 + # DNS query timeout dns_timeout: 5 # How many times to retry DNS queries dns_retries: 1 -# Disable BBOT's smart DNS wildcard handling for select domains +# Completely disable BBOT's DNS wildcard detection +dns_wildcard_disable: False +# Disable BBOT's DNS wildcard detection for select domains dns_wildcard_ignore: [] # How many sanity checks to make when verifying wildcard DNS # Increase this value if BBOT's wildcard detection isn't working @@ -95,6 +98,7 @@ dns_abort_threshold: 50 dns_filter_ptrs: true # Enable/disable debug messages for dns queries dns_debug: false + # Whether to verify SSL certificates ssl_verify: false # How many scan results to keep before cleaning up the older ones diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index 12e5608d9..104ae0ad7 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -10,6 +10,8 @@ class DNS(HookModule): watched_events = ["*"] + options = {"max_event_handlers": 25} + options_desc = {"max_event_handlers": "Number of concurrent DNS workers"} _priority = 1 _max_event_handlers = 25 scope_distance_modifier = None diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 0355656b8..afc5c1967 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -194,6 +194,20 @@ async def test_wildcards(bbot_scanner): assert wildcard_event2.data == "_wildcard.github.io" assert wildcard_event3.data == "github.io" + # dns resolve distance + event_distance_0 = scan.make_event("8.8.8.8", module=scan._make_dummy_module_dns("PTR"), source=scan.root_event) + assert event_distance_0.dns_resolve_distance == 0 + event_distance_1 = scan.make_event( + "evilcorp.com", module=scan._make_dummy_module_dns("A"), source=event_distance_0 + ) + assert event_distance_1.dns_resolve_distance == 1 + event_distance_2 = scan.make_event("1.2.3.4", module=scan._make_dummy_module_dns("PTR"), source=event_distance_1) + assert event_distance_2.dns_resolve_distance == 1 + event_distance_3 = scan.make_event( + "evilcorp.org", module=scan._make_dummy_module_dns("A"), source=event_distance_2 + ) + assert event_distance_3.dns_resolve_distance == 2 + from bbot.scanner import Scanner # test with full scan From 37d1a6eb22c8488e28308f40b26eee333f5cc4bc Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 14:04:30 -0400 Subject: [PATCH 138/171] fix bucket tests --- .../test_step_2/module_tests/test_module_bucket_amazon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py b/bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py index 6d58dd36f..37ce77c5a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py +++ b/bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py @@ -34,7 +34,7 @@ def module_name(self): @property def modules_overrides(self): - return ["excavate", "speculate", "httpx", self.module_name] + return ["excavate", "speculate", "httpx", self.module_name, "cloud"] def url_setup(self): self.url_1 = f"https://{self.random_bucket_1}/" @@ -71,7 +71,7 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): # make sure buckets were excavated assert any( - e.type == "STORAGE_BUCKET" and str(e.module) == f"{self.provider}_cloud" for e in events + e.type == "STORAGE_BUCKET" and str(e.module) == f"cloud_{self.provider}" for e in events ), f'bucket not found for module "{self.module_name}"' # make sure open buckets were found if module_test.module.supports_open_check: From 56a1557a566e8ee76ce248b598a7bbf9b689e3a4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 Apr 2024 15:48:05 -0400 Subject: [PATCH 139/171] fix csv tests --- bbot/modules/internal/dns.py | 5 ++++- bbot/test/test_step_2/module_tests/test_module_csv.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index 104ae0ad7..c55e7c5eb 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -63,7 +63,10 @@ async def handle_event(self, event, kwargs): if event_is_ip: rdtypes_to_resolve = ["PTR"] else: - rdtypes_to_resolve = all_rdtypes + if self.dns_resolution: + rdtypes_to_resolve = all_rdtypes + else: + rdtypes_to_resolve = ("A", "AAAA", "CNAME") # if missing from cache, do DNS resolution queries = [(event_host, rdtype) for rdtype in rdtypes_to_resolve] diff --git a/bbot/test/test_step_2/module_tests/test_module_csv.py b/bbot/test/test_step_2/module_tests/test_module_csv.py index fc180d481..0d6e326a9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_csv.py +++ b/bbot/test/test_step_2/module_tests/test_module_csv.py @@ -2,6 +2,9 @@ class TestCSV(ModuleTestBase): + async def setup_after_prep(self, module_test): + await module_test.mock_dns({}) + def check(self, module_test, events): csv_file = module_test.scan.home / "output.csv" with open(csv_file) as f: From ffada9e211c48ce2b2db07a73e0ac5e66a5778d2 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 16 Apr 2024 16:45:14 -0400 Subject: [PATCH 140/171] rename HookModule --> InterceptModule --- bbot/modules/base.py | 12 +- bbot/modules/internal/cloud.py | 4 +- bbot/modules/internal/dns.py | 4 +- bbot/scanner/manager.py | 12 +- .../test_manager_scope_accuracy.py | 2 +- .../module_tests/test_module_dns.py | 1 - poetry.lock | 438 +++++------------- pyproject.toml | 1 - 8 files changed, 138 insertions(+), 336 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 0f8867201..26332aca6 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1402,7 +1402,17 @@ def critical(self, *args, trace=True, **kwargs): self.trace() -class HookModule(BaseModule): +class InterceptModule(BaseModule): + """ + An Intercept Module is a special type of high-priority module that gets early access to events. + + If you want your module to tag or modify an event before it's distributed to the scan, it should + probably be an intercept module. + + Examples of intercept modules include `dns` (for DNS resolution and wildcard detection) + and `cloud` (for detection and tagging of cloud assets). + """ + accept_dupes = True suppress_dupes = False _hook = True diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index bf69baa35..6bfceacff 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -1,7 +1,7 @@ -from bbot.modules.base import HookModule +from bbot.modules.base import InterceptModule -class cloud(HookModule): +class cloud(InterceptModule): watched_events = ["*"] meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"} scope_distance_modifier = 1 diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index c55e7c5eb..c3db74891 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -3,12 +3,12 @@ from cachetools import LRUCache from bbot.errors import ValidationError -from bbot.modules.base import HookModule +from bbot.modules.base import InterceptModule from bbot.core.helpers.dns.engine import all_rdtypes from bbot.core.helpers.async_helpers import NamedLock -class DNS(HookModule): +class DNS(InterceptModule): watched_events = ["*"] options = {"max_event_handlers": 25} options_desc = {"max_event_handlers": "Number of concurrent DNS workers"} diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 2fd218861..dd01d5879 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -2,14 +2,16 @@ import logging from contextlib import suppress -from bbot.modules.base import HookModule +from bbot.modules.base import InterceptModule log = logging.getLogger("bbot.scanner.manager") -class ScanIngress(HookModule): +class ScanIngress(InterceptModule): """ - This is always the first hook module in the chain, responsible for basic scope checks + This is always the first intercept module in the chain, responsible for basic scope checks + + It has its own incoming queue, but will also pull events from modules' outgoing queues """ watched_events = ["*"] @@ -137,9 +139,9 @@ def is_incoming_duplicate(self, event, add=False): return False -class ScanEgress(HookModule): +class ScanEgress(InterceptModule): """ - This is always the last hook module in the chain, responsible for executing and acting on the + This is always the last intercept module in the chain, responsible for executing and acting on the `abort_if` and `on_success_callback` functions. """ diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 9ee2c72e5..bc79a0029 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -742,7 +742,7 @@ def custom_setup(scan): assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == False and e.scope_distance == 1 and str(e.module) == "sslcert"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "www.bbottest.notreal:9999"]) - assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "bbottest.notreal"]) + assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "bbottest.notreal"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) # sslcert with out-of-scope chain diff --git a/bbot/test/test_step_2/module_tests/test_module_dns.py b/bbot/test/test_step_2/module_tests/test_module_dns.py index 0351ac44e..d74b62351 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dns.py +++ b/bbot/test/test_step_2/module_tests/test_module_dns.py @@ -18,7 +18,6 @@ async def setup_after_prep(self, module_test): ) def check(self, module_test, events): - self.log.critical(events) assert 1 == len( [ e diff --git a/poetry.lock b/poetry.lock index ebe1b7013..00ee05715 100644 --- a/poetry.lock +++ b/poetry.lock @@ -584,21 +584,24 @@ wmi = ["wmi (>=1.5.1)"] [[package]] name = "docutils" -version = "0.20.1" +version = "0.21.1" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = "*" -files = [] +python-versions = ">=3.9" +files = [ + {file = "docutils-0.21.1-py3-none-any.whl", hash = "sha256:14c8d34a55b46c88f9f714adb29cefbdd69fb82f3fef825e59c5faab935390d8"}, + {file = "docutils-0.21.1.tar.gz", hash = "sha256:65249d8a5345bc95e0f40f280ba63c98eb24de35c6c8f5b662e3e8948adea83f"}, +] [[package]] name = "dunamai" -version = "1.19.2" +version = "1.20.0" description = "Dynamic version generation" optional = false python-versions = ">=3.5" files = [ - {file = "dunamai-1.19.2-py3-none-any.whl", hash = "sha256:bc126b17571a44d68ed826cec596e0f61dc01edca8b21486f70014936a5d44f2"}, - {file = "dunamai-1.19.2.tar.gz", hash = "sha256:3be4049890763e19b8df1d52960dbea60b3e263eb0c96144a677ae0633734d2e"}, + {file = "dunamai-1.20.0-py3-none-any.whl", hash = "sha256:a2185c227351a52a013c7d7a695d3f3cb6625c3eed14a5295adbbcc7e2f7f8d4"}, + {file = "dunamai-1.20.0.tar.gz", hash = "sha256:c3f1ee64a1e6cc9ebc98adafa944efaccd0db32482d2177e59c1ff6bdf23cd70"}, ] [package.dependencies] @@ -669,13 +672,13 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "griffe" -version = "0.42.1" +version = "0.42.2" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.8" files = [ - {file = "griffe-0.42.1-py3-none-any.whl", hash = "sha256:7e805e35617601355edcac0d3511cedc1ed0cb1f7645e2d336ae4b05bbae7b3b"}, - {file = "griffe-0.42.1.tar.gz", hash = "sha256:57046131384043ed078692b85d86b76568a686266cc036b9b56b704466f803ce"}, + {file = "griffe-0.42.2-py3-none-any.whl", hash = "sha256:bf9a09d7e9dcc3aca6a2c7ab4f63368c19e882f58c816fbd159bea613daddde3"}, + {file = "griffe-0.42.2.tar.gz", hash = "sha256:d5547b7a1a0786f84042379a5da8bd97c11d0464d4de3d7510328ebce5fda772"}, ] [package.dependencies] @@ -692,32 +695,6 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] -[[package]] -name = "h2" -version = "4.1.0" -description = "HTTP/2 State-Machine based protocol implementation" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "h2-4.1.0-py3-none-any.whl", hash = "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d"}, - {file = "h2-4.1.0.tar.gz", hash = "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"}, -] - -[package.dependencies] -hpack = ">=4.0,<5" -hyperframe = ">=6.0,<7" - -[[package]] -name = "hpack" -version = "4.0.0" -description = "Pure-Python HPACK header compression" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "hpack-4.0.0-py3-none-any.whl", hash = "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c"}, - {file = "hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"}, -] - [[package]] name = "httpcore" version = "1.0.5" @@ -763,17 +740,6 @@ cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -[[package]] -name = "hyperframe" -version = "6.0.1" -description = "HTTP/2 framing layer for Python" -optional = false -python-versions = ">=3.6.1" -files = [ - {file = "hyperframe-6.0.1-py3-none-any.whl", hash = "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15"}, - {file = "hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"}, -] - [[package]] name = "identify" version = "2.5.35" @@ -861,17 +827,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "kiss-headers" -version = "2.4.3" -description = "Object-oriented HTTP and IMAP (structured) headers." -optional = false -python-versions = ">=3.7" -files = [ - {file = "kiss_headers-2.4.3-py3-none-any.whl", hash = "sha256:9d800b77532068e8748be9f96f30eaeb547cdc5345e4689ddf07b77071256239"}, - {file = "kiss_headers-2.4.3.tar.gz", hash = "sha256:70c689ce167ac83146f094ea916b40a3767d67c2e05a4cb95b0fd2e33bf243f1"}, -] - [[package]] name = "libsass" version = "0.23.0" @@ -1257,13 +1212,13 @@ mkdocs = ">=1.1" [[package]] name = "mkdocs-material" -version = "9.5.17" +version = "9.5.18" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.17-py3-none-any.whl", hash = "sha256:14a2a60119a785e70e765dd033e6211367aca9fc70230e577c1cf6a326949571"}, - {file = "mkdocs_material-9.5.17.tar.gz", hash = "sha256:06ae1275a72db1989cf6209de9e9ecdfbcfdbc24c58353877b2bb927dbe413e4"}, + {file = "mkdocs_material-9.5.18-py3-none-any.whl", hash = "sha256:1e0e27fc9fe239f9064318acf548771a4629d5fd5dfd45444fd80a953fe21eb4"}, + {file = "mkdocs_material-9.5.18.tar.gz", hash = "sha256:a43f470947053fa2405c33995f282d24992c752a50114f23f30da9d8d0c57e62"}, ] [package.dependencies] @@ -1349,30 +1304,6 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] -[[package]] -name = "niquests" -version = "3.5.5" -description = "Niquests is a simple, yet elegant, HTTP library. It is a drop-in replacement for Requests, which is under feature freeze." -optional = false -python-versions = ">=3.7" -files = [ - {file = "niquests-3.5.5-py3-none-any.whl", hash = "sha256:bd134c7cbc414661840e73bebe0b766c16321558b3c444efb3f63aad9189e308"}, - {file = "niquests-3.5.5.tar.gz", hash = "sha256:5b52183cd4ee16f360de1e5b97bc266b933e8603320102d10d17f68a95e926ba"}, -] - -[package.dependencies] -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -kiss-headers = ">=2,<4" -urllib3-future = ">=2.7.900,<3" -wassima = ">=1.0.1,<2" - -[package.extras] -http3 = ["urllib3-future[qh3]"] -ocsp = ["cryptography (>=41.0.0,<43.0.0)"] -socks = ["urllib3-future[socks]"] -speedups = ["orjson (>=3,<4)", "urllib3-future[brotli,zstd]"] - [[package]] name = "nodeenv" version = "1.8.0" @@ -2152,147 +2083,106 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} -[[package]] -name = "qh3" -version = "0.15.1" -description = "An implementation of QUIC and HTTP/3" -optional = false -python-versions = ">=3.7" -files = [ - {file = "qh3-0.15.1-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fe8f15e9fe5850508188ce38bdc89bda03d1a99ce3c2fbde6ee02d1d91edc557"}, - {file = "qh3-0.15.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:114d04dd51d3d9eca76ce804fea60ccb0fcbe84be08dcca70f32e30e5736aa00"}, - {file = "qh3-0.15.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:265240539a630cf458f3651f08bd07e4d46b2bf941a25e7f594321401701b30d"}, - {file = "qh3-0.15.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1074ee0e30fe825b60bd113767b56dcfe2f155e79f893d5180d4fd2adebaa1de"}, - {file = "qh3-0.15.1-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0afd9e7b90c90ff3e8c8e376020e3753936da0ce8db57ebb9fc95a50ba7e015d"}, - {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b7c4df89b03f90f67e372693c70f357dabc18908cb07dab21aa550c4f777017b"}, - {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:3923bb17dbdf91f060cb3b04cb8c2e3bf74d528a26f4c0e5365e311bade33b58"}, - {file = "qh3-0.15.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:87b61b59e3c692b70384430ccf634a228c54bb38ee6d974d76a7b086b356ecad"}, - {file = "qh3-0.15.1-cp37-abi3-win32.whl", hash = "sha256:3d02314850b0c8a5cd39015b9f5e5b21d54980702e3e80dcfc6aa7b983d7494a"}, - {file = "qh3-0.15.1-cp37-abi3-win_amd64.whl", hash = "sha256:1a0305b389cec13af879dee32c6584cff45a52865456e6645d84023ed8442d67"}, - {file = "qh3-0.15.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8c000a32d2d3dadf252a55d71f676011f02c0e529024176d35e53122293d8a54"}, - {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9562c2648a0e468cc3c97e77c658c0b9db288e29cfc79d52220e50ddcfac9fe9"}, - {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71ab5d62606556c0ba2b1f3bf118dcb2d6f0236add792ffba42845a741abe498"}, - {file = "qh3-0.15.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:592bd246325090ffe8324761808713b1c99c7b7cae37ec4bd2841d0054729422"}, - {file = "qh3-0.15.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f68ac19161aaef887351f2e8df1972d91726ade69105b4ae1653ab0e70a18536"}, - {file = "qh3-0.15.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a842e65e57f8092f1fa185b1dc95556b1b695f06a4eb48dc9c07f018bd7a7ec"}, - {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bca27698ad110fabda026f844f453b1ac1a1e2d86729846f5be0cdc9e7df419"}, - {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c87a0613efbc3d353a76a917044270caf43198890ffe702b3cbe9b44065c45e"}, - {file = "qh3-0.15.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77673a9b02e19c4f81e419efa2aa4040dec10f0a6158788196d8b5ef6aafb0d9"}, - {file = "qh3-0.15.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:087da39ebd5a8608e8df0892860b4fdcd4ff83753d7312cead490de6f1bce504"}, - {file = "qh3-0.15.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:561ba4d84e617ecc0d7506f532da2814e672a06cdcb903209616f00c5da74c14"}, - {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec733c6a4da5ecf4448434562aba617ecfabbdef0a58df812684db7d03000070"}, - {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74cc03a94e605820f3c5882e47388e8d2d8616d51db57a6e5120d9f2344dc04a"}, - {file = "qh3-0.15.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:939253ceaf5664c4e90f6317f0097839b6c8af627bb5905181f4fcbbc209c395"}, - {file = "qh3-0.15.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:969582d286be3e468ff5e53cdf2a5f47a942ea370f870a0276c4235a7ed13a71"}, - {file = "qh3-0.15.1.tar.gz", hash = "sha256:816c787f68855a28aa703be54956b21ff258e1650978a06b98a23bbf252cbe7e"}, -] - -[package.dependencies] -cryptography = ">=41.0.0,<43" - -[package.extras] -dev = ["coverage[toml] (>=7.2.2)"] - [[package]] name = "regex" -version = "2023.12.25" +version = "2024.4.16" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.7" files = [ - {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5"}, - {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8"}, - {file = "regex-2023.12.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5"}, - {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd"}, - {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704"}, - {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1"}, - {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392"}, - {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423"}, - {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f"}, - {file = "regex-2023.12.25-cp310-cp310-win32.whl", hash = "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630"}, - {file = "regex-2023.12.25-cp310-cp310-win_amd64.whl", hash = "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105"}, - {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6"}, - {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97"}, - {file = "regex-2023.12.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887"}, - {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb"}, - {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c"}, - {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b"}, - {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa"}, - {file = "regex-2023.12.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7"}, - {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0"}, - {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe"}, - {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80"}, - {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd"}, - {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4"}, - {file = "regex-2023.12.25-cp311-cp311-win32.whl", hash = "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87"}, - {file = "regex-2023.12.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f"}, - {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715"}, - {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d"}, - {file = "regex-2023.12.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a"}, - {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a"}, - {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5"}, - {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060"}, - {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3"}, - {file = "regex-2023.12.25-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9"}, - {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f"}, - {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c"}, - {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457"}, - {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf"}, - {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d"}, - {file = "regex-2023.12.25-cp312-cp312-win32.whl", hash = "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5"}, - {file = "regex-2023.12.25-cp312-cp312-win_amd64.whl", hash = "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232"}, - {file = "regex-2023.12.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f"}, - {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8"}, - {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a"}, - {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39"}, - {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b"}, - {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347"}, - {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39"}, - {file = "regex-2023.12.25-cp37-cp37m-win32.whl", hash = "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c"}, - {file = "regex-2023.12.25-cp37-cp37m-win_amd64.whl", hash = "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445"}, - {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53"}, - {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64"}, - {file = "regex-2023.12.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988"}, - {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861"}, - {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc"}, - {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4"}, - {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360"}, - {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756"}, - {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2"}, - {file = "regex-2023.12.25-cp38-cp38-win32.whl", hash = "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb"}, - {file = "regex-2023.12.25-cp38-cp38-win_amd64.whl", hash = "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697"}, - {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31"}, - {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7"}, - {file = "regex-2023.12.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6"}, - {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923"}, - {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d"}, - {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca"}, - {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5"}, - {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f"}, - {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20"}, - {file = "regex-2023.12.25-cp39-cp39-win32.whl", hash = "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9"}, - {file = "regex-2023.12.25-cp39-cp39-win_amd64.whl", hash = "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91"}, - {file = "regex-2023.12.25.tar.gz", hash = "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5"}, + {file = "regex-2024.4.16-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb83cc090eac63c006871fd24db5e30a1f282faa46328572661c0a24a2323a08"}, + {file = "regex-2024.4.16-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c91e1763696c0eb66340c4df98623c2d4e77d0746b8f8f2bee2c6883fd1fe18"}, + {file = "regex-2024.4.16-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:10188fe732dec829c7acca7422cdd1bf57d853c7199d5a9e96bb4d40db239c73"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:956b58d692f235cfbf5b4f3abd6d99bf102f161ccfe20d2fd0904f51c72c4c66"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a70b51f55fd954d1f194271695821dd62054d949efd6368d8be64edd37f55c86"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c02fcd2bf45162280613d2e4a1ca3ac558ff921ae4e308ecb307650d3a6ee51"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ed75ea6892a56896d78f11006161eea52c45a14994794bcfa1654430984b22"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd727ad276bb91928879f3aa6396c9a1d34e5e180dce40578421a691eeb77f47"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7cbc5d9e8a1781e7be17da67b92580d6ce4dcef5819c1b1b89f49d9678cc278c"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:78fddb22b9ef810b63ef341c9fcf6455232d97cfe03938cbc29e2672c436670e"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:445ca8d3c5a01309633a0c9db57150312a181146315693273e35d936472df912"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:95399831a206211d6bc40224af1c635cb8790ddd5c7493e0bd03b85711076a53"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:7731728b6568fc286d86745f27f07266de49603a6fdc4d19c87e8c247be452af"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4facc913e10bdba42ec0aee76d029aedda628161a7ce4116b16680a0413f658a"}, + {file = "regex-2024.4.16-cp310-cp310-win32.whl", hash = "sha256:911742856ce98d879acbea33fcc03c1d8dc1106234c5e7d068932c945db209c0"}, + {file = "regex-2024.4.16-cp310-cp310-win_amd64.whl", hash = "sha256:e0a2df336d1135a0b3a67f3bbf78a75f69562c1199ed9935372b82215cddd6e2"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1210365faba7c2150451eb78ec5687871c796b0f1fa701bfd2a4a25420482d26"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9ab40412f8cd6f615bfedea40c8bf0407d41bf83b96f6fc9ff34976d6b7037fd"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fd80d1280d473500d8086d104962a82d77bfbf2b118053824b7be28cd5a79ea5"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb966fdd9217e53abf824f437a5a2d643a38d4fd5fd0ca711b9da683d452969"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20b7a68444f536365af42a75ccecb7ab41a896a04acf58432db9e206f4e525d6"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b74586dd0b039c62416034f811d7ee62810174bb70dffcca6439f5236249eb09"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c8290b44d8b0af4e77048646c10c6e3aa583c1ca67f3b5ffb6e06cf0c6f0f89"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2d80a6749724b37853ece57988b39c4e79d2b5fe2869a86e8aeae3bbeef9eb0"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3a1018e97aeb24e4f939afcd88211ace472ba566efc5bdf53fd8fd7f41fa7170"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8d015604ee6204e76569d2f44e5a210728fa917115bef0d102f4107e622b08d5"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:3d5ac5234fb5053850d79dd8eb1015cb0d7d9ed951fa37aa9e6249a19aa4f336"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:0a38d151e2cdd66d16dab550c22f9521ba79761423b87c01dae0a6e9add79c0d"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:159dc4e59a159cb8e4e8f8961eb1fa5d58f93cb1acd1701d8aff38d45e1a84a6"}, + {file = "regex-2024.4.16-cp311-cp311-win32.whl", hash = "sha256:ba2336d6548dee3117520545cfe44dc28a250aa091f8281d28804aa8d707d93d"}, + {file = "regex-2024.4.16-cp311-cp311-win_amd64.whl", hash = "sha256:8f83b6fd3dc3ba94d2b22717f9c8b8512354fd95221ac661784df2769ea9bba9"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:80b696e8972b81edf0af2a259e1b2a4a661f818fae22e5fa4fa1a995fb4a40fd"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d61ae114d2a2311f61d90c2ef1358518e8f05eafda76eaf9c772a077e0b465ec"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ba6745440b9a27336443b0c285d705ce73adb9ec90e2f2004c64d95ab5a7598"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295004b2dd37b0835ea5c14a33e00e8cfa3c4add4d587b77287825f3418d310"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aba818dcc7263852aabb172ec27b71d2abca02a593b95fa79351b2774eb1d2b"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0800631e565c47520aaa04ae38b96abc5196fe8b4aa9bd864445bd2b5848a7a"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08dea89f859c3df48a440dbdcd7b7155bc675f2fa2ec8c521d02dc69e877db70"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eeaa0b5328b785abc344acc6241cffde50dc394a0644a968add75fcefe15b9d4"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4e819a806420bc010489f4e741b3036071aba209f2e0989d4750b08b12a9343f"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c2d0e7cbb6341e830adcbfa2479fdeebbfbb328f11edd6b5675674e7a1e37730"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:91797b98f5e34b6a49f54be33f72e2fb658018ae532be2f79f7c63b4ae225145"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:d2da13568eff02b30fd54fccd1e042a70fe920d816616fda4bf54ec705668d81"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:370c68dc5570b394cbaadff50e64d705f64debed30573e5c313c360689b6aadc"}, + {file = "regex-2024.4.16-cp312-cp312-win32.whl", hash = "sha256:904c883cf10a975b02ab3478bce652f0f5346a2c28d0a8521d97bb23c323cc8b"}, + {file = "regex-2024.4.16-cp312-cp312-win_amd64.whl", hash = "sha256:785c071c982dce54d44ea0b79cd6dfafddeccdd98cfa5f7b86ef69b381b457d9"}, + {file = "regex-2024.4.16-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2f142b45c6fed48166faeb4303b4b58c9fcd827da63f4cf0a123c3480ae11fb"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e87ab229332ceb127a165612d839ab87795972102cb9830e5f12b8c9a5c1b508"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81500ed5af2090b4a9157a59dbc89873a25c33db1bb9a8cf123837dcc9765047"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b340cccad138ecb363324aa26893963dcabb02bb25e440ebdf42e30963f1a4e0"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c72608e70f053643437bd2be0608f7f1c46d4022e4104d76826f0839199347a"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a01fe2305e6232ef3e8f40bfc0f0f3a04def9aab514910fa4203bafbc0bb4682"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:03576e3a423d19dda13e55598f0fd507b5d660d42c51b02df4e0d97824fdcae3"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:549c3584993772e25f02d0656ac48abdda73169fe347263948cf2b1cead622f3"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:34422d5a69a60b7e9a07a690094e824b66f5ddc662a5fc600d65b7c174a05f04"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:5f580c651a72b75c39e311343fe6875d6f58cf51c471a97f15a938d9fe4e0d37"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3399dd8a7495bbb2bacd59b84840eef9057826c664472e86c91d675d007137f5"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d1f86f3f4e2388aa3310b50694ac44daefbd1681def26b4519bd050a398dc5a"}, + {file = "regex-2024.4.16-cp37-cp37m-win32.whl", hash = "sha256:dd5acc0a7d38fdc7a3a6fd3ad14c880819008ecb3379626e56b163165162cc46"}, + {file = "regex-2024.4.16-cp37-cp37m-win_amd64.whl", hash = "sha256:ba8122e3bb94ecda29a8de4cf889f600171424ea586847aa92c334772d200331"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:743deffdf3b3481da32e8a96887e2aa945ec6685af1cfe2bcc292638c9ba2f48"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7571f19f4a3fd00af9341c7801d1ad1967fc9c3f5e62402683047e7166b9f2b4"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:df79012ebf6f4efb8d307b1328226aef24ca446b3ff8d0e30202d7ebcb977a8c"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e757d475953269fbf4b441207bb7dbdd1c43180711b6208e129b637792ac0b93"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4313ab9bf6a81206c8ac28fdfcddc0435299dc88cad12cc6305fd0e78b81f9e4"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d83c2bc678453646f1a18f8db1e927a2d3f4935031b9ad8a76e56760461105dd"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9df1bfef97db938469ef0a7354b2d591a2d438bc497b2c489471bec0e6baf7c4"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62120ed0de69b3649cc68e2965376048793f466c5a6c4370fb27c16c1beac22d"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c2ef6f7990b6e8758fe48ad08f7e2f66c8f11dc66e24093304b87cae9037bb4a"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8fc6976a3395fe4d1fbeb984adaa8ec652a1e12f36b56ec8c236e5117b585427"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:03e68f44340528111067cecf12721c3df4811c67268b897fbe695c95f860ac42"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ec7e0043b91115f427998febaa2beb82c82df708168b35ece3accb610b91fac1"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c21fc21a4c7480479d12fd8e679b699f744f76bb05f53a1d14182b31f55aac76"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:12f6a3f2f58bb7344751919a1876ee1b976fe08b9ffccb4bbea66f26af6017b9"}, + {file = "regex-2024.4.16-cp38-cp38-win32.whl", hash = "sha256:479595a4fbe9ed8f8f72c59717e8cf222da2e4c07b6ae5b65411e6302af9708e"}, + {file = "regex-2024.4.16-cp38-cp38-win_amd64.whl", hash = "sha256:0534b034fba6101611968fae8e856c1698da97ce2efb5c2b895fc8b9e23a5834"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a7ccdd1c4a3472a7533b0a7aa9ee34c9a2bef859ba86deec07aff2ad7e0c3b94"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6f2f017c5be19984fbbf55f8af6caba25e62c71293213f044da3ada7091a4455"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:803b8905b52de78b173d3c1e83df0efb929621e7b7c5766c0843704d5332682f"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:684008ec44ad275832a5a152f6e764bbe1914bea10968017b6feaecdad5736e0"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65436dce9fdc0aeeb0a0effe0839cb3d6a05f45aa45a4d9f9c60989beca78b9c"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea355eb43b11764cf799dda62c658c4d2fdb16af41f59bb1ccfec517b60bcb07"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c1165f3809ce7774f05cb74e5408cd3aa93ee8573ae959a97a53db3ca3180d"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cccc79a9be9b64c881f18305a7c715ba199e471a3973faeb7ba84172abb3f317"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00169caa125f35d1bca6045d65a662af0202704489fada95346cfa092ec23f39"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6cc38067209354e16c5609b66285af17a2863a47585bcf75285cab33d4c3b8df"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:23cff1b267038501b179ccbbd74a821ac4a7192a1852d1d558e562b507d46013"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d320b3bf82a39f248769fc7f188e00f93526cc0fe739cfa197868633d44701"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:89ec7f2c08937421bbbb8b48c54096fa4f88347946d4747021ad85f1b3021b3c"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4918fd5f8b43aa7ec031e0fef1ee02deb80b6afd49c85f0790be1dc4ce34cb50"}, + {file = "regex-2024.4.16-cp39-cp39-win32.whl", hash = "sha256:684e52023aec43bdf0250e843e1fdd6febbe831bd9d52da72333fa201aaa2335"}, + {file = "regex-2024.4.16-cp39-cp39-win_amd64.whl", hash = "sha256:e697e1c0238133589e00c244a8b676bc2cfc3ab4961318d902040d099fec7483"}, + {file = "regex-2024.4.16.tar.gz", hash = "sha256:fa454d26f2e87ad661c4f0c5a5fe4cf6aab1e307d1b94f16ffdfcb089ba685c0"}, ] [[package]] @@ -2348,18 +2238,18 @@ test = ["commentjson", "packaging", "pytest"] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2511,28 +2401,6 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "urllib3-future" -version = "2.7.903" -description = "urllib3.future is a powerful HTTP 1.1, 2, and 3 client with both sync and async interfaces" -optional = false -python-versions = ">=3.7" -files = [ - {file = "urllib3_future-2.7.903-py3-none-any.whl", hash = "sha256:04bebce1291c9be9db2b03bb016db56d1f7e3dbe425c7250129552a8ceaf6827"}, - {file = "urllib3_future-2.7.903.tar.gz", hash = "sha256:99e1265c8bb2478d86b8a6c0de991ac275ad58d5e43ac11d980a0dd1cc183804"}, -] - -[package.dependencies] -h11 = ">=0.11.0,<1.0.0" -h2 = ">=4.0.0,<5.0.0" -qh3 = {version = ">=0.14.0,<1.0.0", markers = "(platform_system == \"Darwin\" or platform_system == \"Windows\" or platform_system == \"Linux\") and (platform_python_implementation == \"CPython\" or (platform_python_implementation == \"PyPy\" and python_version >= \"3.8\" and python_version < \"3.11\"))"} - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -qh3 = ["qh3 (>=0.14.0,<1.0.0)"] -socks = ["python-socks (>=2.0,<3.0)"] -zstd = ["zstandard (>=0.18.0)"] - [[package]] name = "virtualenv" version = "20.25.1" @@ -2553,82 +2421,6 @@ platformdirs = ">=3.9.1,<5" docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] -[[package]] -name = "wassima" -version = "1.1.0" -description = "Access your OS root certificates with the atmost ease" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wassima-1.1.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6b67781f7b9483a5dbcb1cabe588ab316f06f7c97a9d60b6981681f790aa16a1"}, - {file = "wassima-1.1.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fb331ab3ff4222ced413a9830c1e9e6a834e7257bfee0043d2f56166ef4aa1cb"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8904e865a2ac81d8160878e7788bc5ee6f4ca6948cf5c5198a83050d68537024"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c414ee94cd1986d7ea3700a6d80efc9ae9b194c37d77396bcfaf927b0d5a620e"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d5d2d1f4f35808a58c8fe7777db14526bd53f77a34b373f070912b2c23f2c3b"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ee1b84222c65f0e2b8ecb6362cc721df1953a0a59e13efc7a4055592fd897f8"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56450dee854ce494003f2be92f2eddb2531c02a456a7866dd32af467672c3b7b"}, - {file = "wassima-1.1.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28897780714f49331fd3e76531ea248df637bbf3e7bf4be175381a92d596c460"}, - {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7528cbfe710af7f9e92cd52296efd7a788466b7cc7fe575b196f604a6ba2281c"}, - {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_armv7l.whl", hash = "sha256:4c3325dff14e796d346e81f90067d054714b99a3d86b6d0a5a76d85bafd2b654"}, - {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:e6609ca3d620792c1dc137efff4c189adee0f13f266ae14515d7de2952159b95"}, - {file = "wassima-1.1.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:60a695e750f9b4fc3cc79cbbbb5e507b8f9715e4135906bb1822126fad1ce5a2"}, - {file = "wassima-1.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:fca891820f7c679d3adc2443d6f85d6201db4badc6b17927d70fa498168d1aea"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:66efd9ee997bfb2311ade7a09f3174d6450a8695ab6b323840539c5826a276c6"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a8550eb00a31eac76a5b5fab3ca2e87cc8d91781191dffa3e133ebf574305321"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df677518d7779fc8a522132c4d96391e0a262dd12bb54ec3937bc8b58f6d3d5"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29d4f6d006ce96c2087c42414ad72ef71bc25bd20ac457dfe89ab2448b0d08e4"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e73264427c6e3f93c7e1b0529960a342a6b4c9c16d17785872a845ee2b0d28f5"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fdbc87751649a23377f279873aae902a38ce44162170edd6b6893d47a259a78"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce49ac61ca651f49c2664003215e259a017d5a1116d669ef331c4930214f53b0"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9522b4905fc75eeaac8518c54362e87d89e83bbefebdb1898a0ef025006e8241"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1b2e419d3075e425ecdcefd486ccd56697dc209e6e2120746477a995392b9402"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1ebe5b0feead8b0457b885f181156574bf9ca88df6fe4cef6ad6b364f02d9e98"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_i686.whl", hash = "sha256:6947c5e2d23383f00199b2cf638d7a090dfe5949bad113387e020b83f2423815"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4420be43f5b4e2b7721080130de565a582299d0d02771c9a7db55366d9c93da5"}, - {file = "wassima-1.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a9fb48413d80d41aa6531a2271516f63c8a1debac016cf8fad6a2fd30aa4486d"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9a4593db51fc02529950158f1217e08c9d62e1299e20a19858f07f80c6d09197"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-macosx_11_0_arm64.whl", hash = "sha256:127aecd895501e79b76114109dba0e4bcf6adcf47169f75d44ecd08b4d983ae7"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7d36eb4e92a348f58182f7f69b0e2fc680ac6605377f5201bac40b303727493"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0e29ace26e79b923d5b0f04c38dff44dc47b9c48684894d8f20841c6ee79760"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab77a0390ba74b7a011918ae5c187e2936cd46f4abffd37c5ff228dbdc4b5e89"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b18821d94eabef23946e21566e7ae7c009ef3a89fe1bc0204e791ba5fdb8ed5"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ef95ad25c04629939d6a6015a798c8b0435cebc0c53cc4b1dabb2a89214a4d8"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ebef47ce05288db306d4f56937f96c48da07afaec014a6ed46ecb17176f874bf"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f0833555f8a334cf1c824f24b07c6b01b13128825d16f7802c4c70d14d2dbe09"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:31afbbe4ea11ea9f92b152e4a61495614bfc0ae3d7c3215a24928144bab79f99"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_i686.whl", hash = "sha256:a759b84855b70922ee31234264ea2f4a058943a38270a18f00fd597f365b4bcb"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa01044ab3ca1f55e2d0d08128a97a68e9022795587627ee9edb3471c72e5df4"}, - {file = "wassima-1.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:08d6cf46441d73335b84c15c4f891adcb59f70701a13ecdee82aead5e0a9b134"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:23b56e0560bd2f35fceab001099bb890d8238fed64e7a0677cacbd1c4d870183"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac1866ee965263e3e024049044e8a5ce905fea2d40e005be03dcd89265fc1e6c"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4285c88f5cb4322318e3c3666d79b923f5317451efc2701011d960774d812675"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1f643f02b0824e789a7c98b9089dfd772a74ceec1a611cf420799f986cadb6bc"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9a7b91256ca429f99beff72dd89b0d5bd6ee1ca8f047138785c5b943eebfb1"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4224cf40a81840618a22164d4002fe5bb9b83cde957ec16e8913996809e705dd"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1006c7510b8495559fc2f1f27a7e49205140eb6b91a91f2c71cd91c2588522ae"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5abfa548d3c7acbf87899fc4af99c5a1fe929cf8cc7a7fd65a825dd88fa37b10"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c5854745eb0fd9243ebbaad46dc1f6f5193dd3f13f12dd19da95877ee2a8d62c"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:9def0580135d80a64aac4761e008d0d82fad5feb9c5028ba9427393144e4a535"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_i686.whl", hash = "sha256:450501472645fe5ea65f1848466ce5a0f2800ed5e13288fa4c210728e2883d24"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:9764c226493e4a9b960156c3657ef7cece430ab8bad0035ebffb0eeb488633cb"}, - {file = "wassima-1.1.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:83792a234431f7fbd06f3370e968b99df430ab3bacdb9ea3318247d55dee3b6c"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:aee6fcfa43ce63691ec30943681e9432ff6cecbd976526c7ec0e5f2aaf85866a"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:31e69da1f3cf1ce4f24dbc4590101d68fcb3e1f715566fe30b6691429e9c1b10"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b574a646498b191bc8974524458d85bc55335992dc8ea7cddcb09ec58c01d4"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3112316434fd3ed3cfb1eac4998f54ed46d07a36172d18d543c0815a98e0d51"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:18e6f92114f878ea26fea7a10af255a6aadfddb1600f20fdfe96d65598e62501"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09d54c87ce23ec2332f2acefc030ae3f4262b94cb1f0c613c8d2e30c297d12d7"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9d953b261b7b64072fbed7b4bf4441f7910d8247387f29cc82f8c314f7acf39"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8ef5a3d129997147f5475c276bc79da14ac59a8f614f07634e2aac5d9b2f94"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e9a6da09d6a03c0c8ec3f5c6b7fa5061f051d67a0e0f0ec1518d2bd76efb7535"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:39d65b2beb0eb17a92cdf859d8e9146a15f8d7f35ab95602780a3ac078069e7e"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_i686.whl", hash = "sha256:e5ed0411e3a14e9352ff83e47952df03b7c8915f9fd4c9fb0888a80ac2759dcf"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:8f5869858975755d90e5505d3a7e2ac687cd09a348bc48137fd5b270969bd7a0"}, - {file = "wassima-1.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:39742c4884b7d1b3314064895e10345b96c7cab0a8f622e65f7beea89c0de4d2"}, - {file = "wassima-1.1.0-py3-none-any.whl", hash = "sha256:d250b77c1964c03f010a271fdd0cad3e14af250fb15cc3a729f23ee1e5922f69"}, - {file = "wassima-1.1.0.tar.gz", hash = "sha256:0ae03025ec07c0491e2d1a499d404eb66180c226f403451042190294f6ec7f06"}, -] - [[package]] name = "watchdog" version = "4.0.0" @@ -2821,4 +2613,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "38517d808d6bc20a9e2c8597b4024707537f2a92d1f75c67a5ed3477c139418b" +content-hash = "1532c2dc5846395a46766fead9f3c29223369ba11025b04e4eebec508fe0d8da" diff --git a/pyproject.toml b/pyproject.toml index cb8766c38..53dfe2131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ cachetools = "^5.3.2" socksio = "^1.0.0" jinja2 = "^3.1.3" pyzmq = "^25.1.2" -niquests = "^3.5.5" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From d90b2b16b24fa6cfaa0f43dbc37487f0cdad9a49 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 16 Apr 2024 17:45:09 -0400 Subject: [PATCH 141/171] let there be shared module dependencies --- bbot/core/helpers/depsinstaller/installer.py | 14 +++ bbot/core/modules.py | 30 ++++++- bbot/core/shared_deps.py | 95 ++++++++++++++++++++ bbot/defaults.yml | 5 ++ bbot/modules/deadly/ffuf.py | 14 +-- bbot/modules/ffuf_shortnames.py | 12 +-- bbot/modules/gowitness.py | 39 +------- bbot/modules/massdns.py | 36 +------- 8 files changed, 144 insertions(+), 101 deletions(-) create mode 100644 bbot/core/shared_deps.py diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 8b9d2ae2b..c386b6c3b 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -148,6 +148,20 @@ async def install_module(self, module): if deps_pip: success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints) + # shared/common + deps_common = preloaded["deps"]["common"] + if deps_common: + for dep_common in deps_common: + if self.setup_status.get(dep_common, False) == True: + log.critical( + f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed' + ) + continue + ansible_tasks = self.preset.module_loader._shared_deps[dep_common] + result = self.tasks(module, ansible_tasks) + self.setup_status[dep_common] = result + success &= result + return success async def pip_install(self, packages, constraints=None): diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 2f3ce445c..508836b2a 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -13,9 +13,11 @@ from contextlib import suppress from bbot.core import CORE +from bbot.errors import BBOTError from bbot.logger import log_to_stderr from .flags import flag_descriptions +from .shared_deps import SHARED_DEPS from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform, mkdir @@ -43,6 +45,8 @@ class ModuleLoader: def __init__(self): self.core = CORE + self._shared_deps = dict(SHARED_DEPS) + self.__preloaded = {} self._modules = {} self._configs = {} @@ -250,6 +254,7 @@ def configs(self, type=None): def find_and_replace(self, **kwargs): self.__preloaded = search_format_dict(self.__preloaded, **kwargs) + self._shared_deps = search_format_dict(self._shared_deps, **kwargs) def check_type(self, module, type): return self._preloaded[module]["type"] == type @@ -312,6 +317,7 @@ def preload_module(self, module_file): deps_pip_constraints = [] deps_shell = [] deps_apt = [] + deps_common = [] ansible_tasks = [] python_code = open(module_file).read() # take a hash of the code so we can keep track of when it changes @@ -380,6 +386,11 @@ def preload_module(self, module_file): # ansible playbook elif any([target.id == "deps_ansible" for target in class_attr.targets]): ansible_tasks = ast.literal_eval(class_attr.value) + # shared/common module dependencies + if any([target.id == "deps_common" for target in class_attr.targets]): + for dep_common in class_attr.value.elts: + if type(dep_common.value) == str: + deps_common.append(dep_common.value) for task in ansible_tasks: if not "become" in task: @@ -403,13 +414,24 @@ def preload_module(self, module_file): "shell": deps_shell, "apt": deps_apt, "ansible": ansible_tasks, + "common": deps_common, }, "sudo": len(deps_apt) > 0, } - if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any( - x == True for x in search_dict_by_key("ansible_become", ansible_tasks) - ): - preloaded_data["sudo"] = True + ansible_task_list = list(ansible_tasks) + for dep_common in deps_common: + try: + ansible_task_list.extend(self._shared_deps[dep_common]) + except KeyError: + common_choices = ",".join(self._shared_deps) + raise BBOTError( + f'Error while preloading module "{module_file}": No shared dependency named "{dep_common}" (choices: {common_choices})' + ) + for ansible_task in ansible_task_list: + if any(x == True for x in search_dict_by_key("become", ansible_task)) or any( + x == True for x in search_dict_by_key("ansible_become", ansible_tasks) + ): + preloaded_data["sudo"] = True return preloaded_data def load_modules(self, module_names): diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py new file mode 100644 index 000000000..c7d83c4a7 --- /dev/null +++ b/bbot/core/shared_deps.py @@ -0,0 +1,95 @@ +DEP_FFUF = [ + { + "name": "Download ffuf", + "unarchive": { + "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_DEPS_FFUF_VERSION}/ffuf_#{BBOT_DEPS_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz", + "include": "ffuf", + "dest": "#{BBOT_TOOLS}", + "remote_src": True, + }, + } +] + +DEP_MASSDNS = [ + { + "name": "install dev tools", + "package": {"name": ["gcc", "git", "make"], "state": "present"}, + "become": True, + "ignore_errors": True, + }, + { + "name": "Download massdns source code", + "git": { + "repo": "https://github.com/blechschmidt/massdns.git", + "dest": "#{BBOT_TEMP}/massdns", + "single_branch": True, + "version": "master", + }, + }, + { + "name": "Build massdns (Linux)", + "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"}, + "when": "ansible_facts['system'] == 'Linux'", + }, + { + "name": "Build massdns (non-Linux)", + "command": { + "chdir": "#{BBOT_TEMP}/massdns", + "cmd": "make nolinux", + "creates": "#{BBOT_TEMP}/massdns/bin/massdns", + }, + "when": "ansible_facts['system'] != 'Linux'", + }, + { + "name": "Install massdns", + "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"}, + }, +] + +DEP_CHROMIUM = [ + { + "name": "Install Chromium (Non-Debian)", + "package": {"name": "chromium", "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] != 'Debian'", + "ignore_errors": True, + }, + { + "name": "Install Chromium dependencies (Debian)", + "package": { + "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2", + "state": "present", + }, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian'", + "ignore_errors": True, + }, + { + "name": "Get latest Chromium version (Debian)", + "uri": { + "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media", + "return_content": True, + }, + "register": "chromium_version", + "when": "ansible_facts['os_family'] == 'Debian'", + "ignore_errors": True, + }, + { + "name": "Download Chromium (Debian)", + "unarchive": { + "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F{{ chromium_version.content }}%2Fchrome-linux.zip?alt=media", + "remote_src": True, + "dest": "#{BBOT_TOOLS}", + "creates": "#{BBOT_TOOLS}/chrome-linux", + }, + "when": "ansible_facts['os_family'] == 'Debian'", + "ignore_errors": True, + }, +] + +# shared module dependencies -- ffuf, massdns, chromium, etc. +SHARED_DEPS = {} +for var, val in list(locals().items()): + if var.startswith("DEP_") and isinstance(val, list): + var = var.split("_", 1)[-1].lower() + SHARED_DEPS[var] = val diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 9eefb838b..4b9b5210d 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -19,6 +19,11 @@ http_proxy: # Web user-agent user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97 +# Tool dependencies +deps: + ffuf: + version: "2.1.0" + ### WEB SPIDER ### # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed) diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index 8382f1e66..a56c73506 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -17,7 +17,6 @@ class ffuf(BaseModule): "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt", "lines": 5000, "max_depth": 0, - "version": "2.0.0", "extensions": "", } @@ -25,21 +24,10 @@ class ffuf(BaseModule): "wordlist": "Specify wordlist to use when finding directories", "lines": "take only the first N lines from the wordlist when finding directories", "max_depth": "the maximum directory depth to attempt to solve", - "version": "ffuf version", "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)", } - deps_ansible = [ - { - "name": "Download ffuf", - "unarchive": { - "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz", - "include": "ffuf", - "dest": "#{BBOT_TOOLS}", - "remote_src": True, - }, - } - ] + deps_common = ["ffuf"] banned_characters = [" "] diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index ca02da886..cfc58cba4 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -59,17 +59,7 @@ class ffuf_shortnames(ffuf): "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them", } - deps_ansible = [ - { - "name": "Download ffuf", - "unarchive": { - "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz", - "include": "ffuf", - "dest": "#{BBOT_TOOLS}", - "remote_src": True, - }, - } - ] + deps_common = ["ffuf"] in_scope_only = True diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 3271ef93f..ea8663bb7 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -29,45 +29,8 @@ class gowitness(BaseModule): "output_path": "where to save screenshots", "social": "Whether to screenshot social media webpages", } + deps_common = ["chromium"] deps_ansible = [ - { - "name": "Install Chromium (Non-Debian)", - "package": {"name": "chromium", "state": "present"}, - "become": True, - "when": "ansible_facts['os_family'] != 'Debian'", - "ignore_errors": True, - }, - { - "name": "Install Chromium dependencies (Debian)", - "package": { - "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2", - "state": "present", - }, - "become": True, - "when": "ansible_facts['os_family'] == 'Debian'", - "ignore_errors": True, - }, - { - "name": "Get latest Chromium version (Debian)", - "uri": { - "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media", - "return_content": True, - }, - "register": "chromium_version", - "when": "ansible_facts['os_family'] == 'Debian'", - "ignore_errors": True, - }, - { - "name": "Download Chromium (Debian)", - "unarchive": { - "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F{{ chromium_version.content }}%2Fchrome-linux.zip?alt=media", - "remote_src": True, - "dest": "#{BBOT_TOOLS}", - "creates": "#{BBOT_TOOLS}/chrome-linux", - }, - "when": "ansible_facts['os_family'] == 'Debian'", - "ignore_errors": True, - }, { "name": "Download gowitness", "get_url": { diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index cad536dfd..540b84041 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -38,41 +38,7 @@ class massdns(subdomain_enum): "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com", } subdomain_file = None - deps_ansible = [ - { - "name": "install dev tools", - "package": {"name": ["gcc", "git", "make"], "state": "present"}, - "become": True, - "ignore_errors": True, - }, - { - "name": "Download massdns source code", - "git": { - "repo": "https://github.com/blechschmidt/massdns.git", - "dest": "#{BBOT_TEMP}/massdns", - "single_branch": True, - "version": "master", - }, - }, - { - "name": "Build massdns (Linux)", - "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"}, - "when": "ansible_facts['system'] == 'Linux'", - }, - { - "name": "Build massdns (non-Linux)", - "command": { - "chdir": "#{BBOT_TEMP}/massdns", - "cmd": "make nolinux", - "creates": "#{BBOT_TEMP}/massdns/bin/massdns", - }, - "when": "ansible_facts['system'] != 'Linux'", - }, - { - "name": "Install massdns", - "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"}, - }, - ] + deps_common = ["massdns"] reject_wildcards = "strict" _qsize = 10000 From 307aa11518bdece807eb4b91753b10284db70bfe Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 16 Apr 2024 17:53:49 -0400 Subject: [PATCH 142/171] add docker shared dependency --- bbot/core/shared_deps.py | 24 ++++++++++++++++++++++++ bbot/modules/deadly/dastardly.py | 24 +----------------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index c7d83c4a7..751117752 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -10,6 +10,30 @@ } ] +DEP_DOCKER = [ + { + "name": "Check if Docker is already installed", + "command": "docker --version", + "register": "docker_installed", + "ignore_errors": True, + }, + { + "name": "Install Docker (Non-Debian)", + "package": {"name": "docker", "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] != 'Debian' and docker_installed.rc != 0", + }, + { + "name": "Install Docker (Debian)", + "package": { + "name": "docker.io", + "state": "present", + }, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian' and docker_installed.rc != 0", + }, +] + DEP_MASSDNS = [ { "name": "install dev tools", diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py index c419f67d9..2bfd20f4a 100644 --- a/bbot/modules/deadly/dastardly.py +++ b/bbot/modules/deadly/dastardly.py @@ -9,29 +9,7 @@ class dastardly(BaseModule): meta = {"description": "Lightweight web application security scanner"} deps_pip = ["lxml~=4.9.2"] - deps_ansible = [ - { - "name": "Check if Docker is already installed", - "command": "docker --version", - "register": "docker_installed", - "ignore_errors": True, - }, - { - "name": "Install Docker (Non-Debian)", - "package": {"name": "docker", "state": "present"}, - "become": True, - "when": "ansible_facts['os_family'] != 'Debian' and docker_installed.rc != 0", - }, - { - "name": "Install Docker (Debian)", - "package": { - "name": "docker.io", - "state": "present", - }, - "become": True, - "when": "ansible_facts['os_family'] == 'Debian' and docker_installed.rc != 0", - }, - ] + deps_common = ["docker"] per_hostport_only = True async def setup(self): From 22f877ec476d16531d4989465b45c43190549b32 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 16 Apr 2024 17:59:21 -0400 Subject: [PATCH 143/171] hook --> intercept --- bbot/modules/base.py | 8 ++++---- bbot/scanner/manager.py | 18 +++++++++--------- bbot/scanner/scanner.py | 18 +++++++++--------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 26332aca6..c102b138d 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -111,7 +111,7 @@ class BaseModule: _priority = 3 _name = "base" _type = "scan" - _hook = False + _intercept = False def __init__(self, scan): """Initializes a module instance. @@ -1415,7 +1415,7 @@ class InterceptModule(BaseModule): accept_dupes = True suppress_dupes = False - _hook = True + _intercept = True async def _worker(self): async with self.scan._acatch(context=self._worker, unhandled_is_critical=True): @@ -1491,7 +1491,7 @@ async def get_incoming_event(self): async def forward_event(self, event, kwargs): """ - Used for forwarding the event on to the next hook module + Used for forwarding the event on to the next intercept module """ await self.outgoing_event_queue.put((event, kwargs)) @@ -1500,7 +1500,7 @@ async def queue_outgoing_event(self, event, **kwargs): Used by emit_event() to raise new events to the scan """ # if this was a normal module, we'd put it in the outgoing queue - # but because it's a hook module, we need to queue it with the first hook module + # but because it's a intercept module, we need to queue it with the first intercept module await self.scan.ingress_module.queue_event(event, kwargs) async def queue_event(self, event, kwargs=None): diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index dd01d5879..76d7b6028 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -27,7 +27,7 @@ def priority(self): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._module_priority_weights = None - self._non_hook_modules = None + self._non_intercept_modules = None # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules) self.incoming_dup_tracker = set() @@ -95,20 +95,20 @@ async def forward_event(self, event, kwargs): await super().forward_event(event, kwargs) @property - def non_hook_modules(self): - if self._non_hook_modules is None: - self._non_hook_modules = [m for m in self.scan.modules.values() if not m._hook] - return self._non_hook_modules + def non_intercept_modules(self): + if self._non_intercept_modules is None: + self._non_intercept_modules = [m for m in self.scan.modules.values() if not m._intercept] + return self._non_intercept_modules @property def incoming_queues(self): - return [self.incoming_event_queue] + [m.outgoing_event_queue for m in self.non_hook_modules] + return [self.incoming_event_queue] + [m.outgoing_event_queue for m in self.non_intercept_modules] @property def module_priority_weights(self): if not self._module_priority_weights: # we subtract from six because lower priorities == higher weights - priorities = [5] + [6 - m.priority for m in self.non_hook_modules] + priorities = [5] + [6 - m.priority for m in self.non_intercept_modules] self._module_priority_weights = priorities return self._module_priority_weights @@ -213,8 +213,8 @@ async def forward_event(self, event, kwargs): self.scan.word_cloud.absorb_event(event) for mod in self.scan.modules.values(): - # don't distribute events to hook modules - if mod._hook: + # don't distribute events to intercept modules + if mod._intercept: continue acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes graph_important = mod._is_graph_important(event) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 4a8ebf0f6..36428cc53 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -252,14 +252,14 @@ async def _prep(self): # run each module's .setup() method succeeded, hard_failed, soft_failed = await self.setup_modules() - # hook modules get sewn together like human centipede - self.hook_modules = [m for m in self.modules.values() if m._hook] - for i, hook_module in enumerate(self.hook_modules[:-1]): - next_hook_module = self.hook_modules[i + 1] + # intercept modules get sewn together like human centipede + self.intercept_modules = [m for m in self.modules.values() if m._intercept] + for i, intercept_module in enumerate(self.intercept_modules[:-1]): + next_intercept_module = self.intercept_modules[i + 1] self.debug( - f"Setting hook module {hook_module.name}.outgoing_event_queue to next hook module {next_hook_module.name}.incoming_event_queue" + f"Setting intercept module {intercept_module.name}.outgoing_event_queue to next intercept module {next_intercept_module.name}.incoming_event_queue" ) - hook_module._outgoing_event_queue = next_hook_module.incoming_event_queue + intercept_module._outgoing_event_queue = next_intercept_module.incoming_event_queue # abort if there are no output modules num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) @@ -428,8 +428,8 @@ async def setup_modules(self, remove_failed=True): else: self.info(f"Setup soft-failed for {module.name}: {msg}") soft_failed.append(module.name) - if (not status) and (module._hook or remove_failed): - # if a hook module fails setup, we always remove it + if (not status) and (module._intercept or remove_failed): + # if a intercept module fails setup, we always remove it self.modules.pop(module.name) return succeeded, hard_failed, soft_failed @@ -514,7 +514,7 @@ async def load_modules(self): f"Loaded {len(loaded_output_modules):,}/{len(self.preset.output_modules):,} output modules, ({','.join(loaded_output_modules)})" ) - # builtin hook modules + # builtin intercept modules self.ingress_module = ScanIngress(self) self.egress_module = ScanEgress(self) self.modules[self.ingress_module.name] = self.ingress_module From 3df1d638a484dab0eca20e25d01d139117614199 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 13:52:15 -0400 Subject: [PATCH 144/171] fix inconsistency with dns host speculation --- bbot/modules/internal/dns.py | 21 +++++++++++++++------ bbot/modules/output/neo4j.py | 7 +++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index c3db74891..baaefc914 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -3,9 +3,9 @@ from cachetools import LRUCache from bbot.errors import ValidationError -from bbot.modules.base import InterceptModule from bbot.core.helpers.dns.engine import all_rdtypes from bbot.core.helpers.async_helpers import NamedLock +from bbot.modules.base import InterceptModule, BaseModule class DNS(InterceptModule): @@ -16,6 +16,13 @@ class DNS(InterceptModule): _max_event_handlers = 25 scope_distance_modifier = None + class HostModule(BaseModule): + _name = "host" + _type = "internal" + + def _outgoing_dedup_hash(self, event): + return hash((event, self.name, event.always_emit)) + async def setup(self): self.dns_resolution = True # you can disable DNS resolution with either the "dns" or "dns_resolution" config options @@ -29,6 +36,8 @@ async def setup(self): self._event_cache = LRUCache(maxsize=10000) self._event_cache_locks = NamedLock() + self.host_module = self.HostModule(self.scan) + return True @property @@ -138,7 +147,7 @@ async def handle_event(self, event, kwargs): self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") event.scope_distance = 0 - # check for wildcards, only if the event resolves to something isn't an IP + # check for wildcards, only if the event resolves to something that isn't an IP if (not event_is_ip) and (dns_children): if event.scope_distance <= self.scan.scope_search_distance: await self.handle_wildcard_event(event) @@ -165,12 +174,12 @@ async def handle_event(self, event, kwargs): if ( event.host and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") - and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") + and not ((event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate")) ): - source_module = self.scan._make_dummy_module("host", _type="internal") - source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event) + source_event = self.scan.make_event(event.host, "DNS_NAME", module=self.host_module, source=event) # only emit the event if it's not already in the parent chain - if source_event is not None and source_event not in event.get_sources(): + if source_event is not None and (source_event.always_emit or source_event not in event.get_sources()): + self.critical(f"SPECULATING {event.host} FROM {event}") source_event.scope_distance = event.scope_distance if "target" in event.tags: source_event.add_tag("target") diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 2cc083544..2b0548ea9 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -1,3 +1,4 @@ +from contextlib import suppress from neo4j import AsyncGraphDatabase from bbot.modules.output.base import BaseOutputModule @@ -78,5 +79,7 @@ async def merge_event(self, event, id_only=False): return (await result.single()).get("id(_)") async def cleanup(self): - await self.session.close() - await self.driver.close() + with suppress(Exception): + await self.session.close() + with suppress(Exception): + await self.driver.close() From 378ee8ac394da75623d17ff30108e099830b8f8b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 14:08:03 -0400 Subject: [PATCH 145/171] fix tests --- bbot/core/modules.py | 16 ++++++++-------- bbot/modules/deadly/vhost.py | 12 +----------- bbot/test/test_step_1/test_modules_basic.py | 3 ++- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 508836b2a..b9ae83af5 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -336,10 +336,10 @@ def preload_module(self, module_file): if any([target.id == "options" for target in class_attr.targets]): config.update(ast.literal_eval(class_attr.value)) # module options - if any([target.id == "options_desc" for target in class_attr.targets]): + elif any([target.id == "options_desc" for target in class_attr.targets]): options_desc.update(ast.literal_eval(class_attr.value)) # module metadata - if any([target.id == "meta" for target in class_attr.targets]): + elif any([target.id == "meta" for target in class_attr.targets]): meta = ast.literal_eval(class_attr.value) # class attributes that are lists @@ -350,27 +350,27 @@ def preload_module(self, module_file): if type(flag.value) == str: flags.add(flag.value) # watched events - if any([target.id == "watched_events" for target in class_attr.targets]): + elif any([target.id == "watched_events" for target in class_attr.targets]): for event_type in class_attr.value.elts: if type(event_type.value) == str: watched_events.add(event_type.value) # produced events - if any([target.id == "produced_events" for target in class_attr.targets]): + elif any([target.id == "produced_events" for target in class_attr.targets]): for event_type in class_attr.value.elts: if type(event_type.value) == str: produced_events.add(event_type.value) # bbot module dependencies - if any([target.id == "deps_modules" for target in class_attr.targets]): + elif any([target.id == "deps_modules" for target in class_attr.targets]): for dep_module in class_attr.value.elts: if type(dep_module.value) == str: deps_modules.add(dep_module.value) # python dependencies - if any([target.id == "deps_pip" for target in class_attr.targets]): + elif any([target.id == "deps_pip" for target in class_attr.targets]): for dep_pip in class_attr.value.elts: if type(dep_pip.value) == str: deps_pip.append(dep_pip.value) - if any([target.id == "deps_pip_constraints" for target in class_attr.targets]): + elif any([target.id == "deps_pip_constraints" for target in class_attr.targets]): for dep_pip in class_attr.value.elts: if type(dep_pip.value) == str: deps_pip_constraints.append(dep_pip.value) @@ -387,7 +387,7 @@ def preload_module(self, module_file): elif any([target.id == "deps_ansible" for target in class_attr.targets]): ansible_tasks = ast.literal_eval(class_attr.value) # shared/common module dependencies - if any([target.id == "deps_common" for target in class_attr.targets]): + elif any([target.id == "deps_common" for target in class_attr.targets]): for dep_common in class_attr.value.elts: if type(dep_common.value) == str: deps_common.append(dep_common.value) diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index e2908dbbe..cf7be1f67 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -22,17 +22,7 @@ class vhost(ffuf): "lines": "take only the first N lines from the wordlist when finding directories", } - deps_ansible = [ - { - "name": "Download ffuf", - "unarchive": { - "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz", - "include": "ffuf", - "dest": "#{BBOT_TOOLS}", - "remote_src": True, - }, - } - ] + deps_common = ["ffuf"] in_scope_only = True diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 5fc187fe3..03273c0a7 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -108,7 +108,8 @@ async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock): assert type(all_preloaded["massdns"]["config"]["max_resolvers"]) == int assert all_preloaded["sslcert"]["deps"]["pip"] assert all_preloaded["sslcert"]["deps"]["apt"] - assert all_preloaded["massdns"]["deps"]["ansible"] + assert all_preloaded["massdns"]["deps"]["common"] + assert all_preloaded["gowitness"]["deps"]["ansible"] all_flags = set() From d6511debc3db1f6d1a29685abc7727f6aae1efdf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 14:09:38 -0400 Subject: [PATCH 146/171] update poetry.lock --- poetry.lock | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index bdb3e03a1..0a89ff3bc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1978,6 +1978,111 @@ files = [ [package.dependencies] pyyaml = "*" +[[package]] +name = "pyzmq" +version = "25.1.2" +description = "Python bindings for 0MQ" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, + {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, + {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, + {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, + {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, + {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, + {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, + {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, + {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, + {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, + {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, + {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, + {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, +] + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} + [[package]] name = "regex" version = "2024.4.16" @@ -2508,4 +2613,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "2d76a3fa67dcc00e96922a7fcc640f7ba7605ab29c260896587521f258a0b9d0" +content-hash = "daf56ec78cff336b530e48e83f0a854c3356a802e5e0cf9456b7a5adfe962354" From 32145c553d92cfab90ed9c171d5bda7e63a98fec Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 16:38:24 -0400 Subject: [PATCH 147/171] fix attribute error --- bbot/core/helpers/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 14a788f8a..06fc8a91f 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -185,7 +185,8 @@ async def _write_proc_line(proc, chunk): proc.stdin.write(smart_encode(chunk) + b"\n") await proc.stdin.drain() except Exception as e: - command = " ".join([str(s) for s in proc.args]) + proc_args = [str(s) for s in getattr(proc, "args", [])] + command = " ".join(proc_args) log.warning(f"Error writing line to stdin for command: {command}: {e}") log.trace(traceback.format_exc()) From 36e39751f82140cf3b23d48561f4ac8e3fc902c4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 17:19:28 -0400 Subject: [PATCH 148/171] WIP regex optimization --- bbot/core/helpers/helper.py | 71 +++++++++++++++++++++++++++++++ bbot/core/helpers/regexes.py | 2 +- bbot/modules/badsecrets.py | 2 +- bbot/modules/bevigil.py | 2 +- bbot/modules/internal/excavate.py | 13 +++--- bbot/modules/secretsdb.py | 2 +- bbot/modules/sslcert.py | 2 +- bbot/modules/wafw00f.py | 6 +-- bbot/modules/wappalyzer.py | 4 +- bbot/modules/wayback.py | 2 +- bbot/scanner/scanner.py | 44 +------------------ poetry.lock | 2 +- pyproject.toml | 1 + 13 files changed, 92 insertions(+), 61 deletions(-) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index d2bb4bb19..ecf7c7cfc 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -1,7 +1,12 @@ import os +import asyncio import logging +import regex as re from pathlib import Path +import multiprocessing as mp +from functools import partial from cloudcheck import cloud_providers +from concurrent.futures import ProcessPoolExecutor from . import misc from .dns import DNSHelper @@ -65,6 +70,18 @@ def __init__(self, preset): self.mkdir(self.tools_dir) self.mkdir(self.lib_dir) + self._loop = None + + # multiprocessing thread pool + start_method = mp.get_start_method() + if start_method != "spawn": + self.warning(f"Multiprocessing spawn method is set to {start_method}.") + + # we spawn 1 fewer processes than cores + # this helps to avoid locking up the system or competing with the main python process for cpu time + num_processes = max(1, mp.cpu_count() - 1) + self.process_pool = ProcessPoolExecutor(max_workers=num_processes) + self.cloud = cloud_providers self.dns = DNSHelper(self) @@ -73,6 +90,28 @@ def __init__(self, preset): self.word_cloud = WordCloud(self) self.dummy_modules = {} + def ensure_compiled_regex(self, r): + """ + Make sure a regex has been compiled + """ + if not isinstance(r, re.Pattern): + raise ValueError("Regex must be compiled first!") + + async def re_search(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.run_in_executor(compiled_regex.search, *args, **kwargs) + + async def re_findall(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.run_in_executor(compiled_regex.findall, *args, **kwargs) + + async def re_finditer(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.run_in_executor(self._re_finditer, compiled_regex, *args, **kwargs) + + def _re_finditer(self, compiled_regex, *args, **kwargs): + return list(compiled_regex.finditer(*args, **kwargs)) + def interactsh(self, *args, **kwargs): return Interactsh(self, *args, **kwargs) @@ -103,6 +142,38 @@ def config(self): def scan(self): return self.preset.scan + @property + def loop(self): + """ + Get the current event loop + """ + if self._loop is None: + self._loop = asyncio.get_running_loop() + return self._loop + + def run_in_executor(self, callback, *args, **kwargs): + """ + Run a synchronous task in the event loop's default thread pool executor + + Examples: + Execute callback: + >>> result = await self.helpers.run_in_executor(callback_fn, arg1, arg2) + """ + callback = partial(callback, **kwargs) + return self.loop.run_in_executor(None, callback, *args) + + def run_in_executor_mp(self, callback, *args, **kwargs): + """ + Same as run_in_executor() except with a process pool executor + Use only in cases where callback is CPU-bound + + Examples: + Execute callback: + >>> result = await self.helpers.run_in_executor_mp(callback_fn, arg1, arg2) + """ + callback = partial(callback, **kwargs) + return self.loop.run_in_executor(self.process_pool, callback, *args) + @property def in_tests(self): return os.environ.get("BBOT_TESTING", "") == "True" diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 6e80801a6..f5fb78f4c 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -1,4 +1,4 @@ -import re +import regex as re from collections import OrderedDict # for extracting words from strings diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 5626314fe..01cc36ed8 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -33,7 +33,7 @@ async def handle_event(self, event): resp_cookies[c2[0]] = c2[1] if resp_body or resp_cookies: try: - r_list = await self.scan.run_in_executor_mp( + r_list = await self.helpers.run_in_executor_mp( carve_all_modules, body=resp_body, headers=resp_headers, diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 435ceae08..bbf339b08 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -34,7 +34,7 @@ async def handle_event(self, event): if self.urls: urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls) if urls: - for parsed_url in await self.scan.run_in_executor_mp(self.helpers.validators.collapse_urls, urls): + for parsed_url in await self.helpers.run_in_executor_mp(self.helpers.validators.collapse_urls, urls): await self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", source=event) async def request_subdomains(self, query): diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 1af70b051..199a72b5e 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -1,7 +1,7 @@ -import re import html import base64 import jwt as j +import regex as re from urllib.parse import urljoin from bbot.core.helpers.regexes import _email_regex, dns_name_regex @@ -14,6 +14,7 @@ class BaseExtractor: def __init__(self, excavate): self.excavate = excavate + self.helpers = excavate.helpers self.compiled_regexes = {} for rname, r in self.regexes.items(): self.compiled_regexes[rname] = re.compile(r) @@ -29,7 +30,7 @@ async def _search(self, content, event, **kwargs): for name, regex in self.compiled_regexes.items(): # yield to event loop await self.excavate.helpers.sleep(0) - for result in regex.findall(content): + for result in await self.helpers.re_findall(regex, content): yield result, name async def report(self, result, name, event): @@ -39,14 +40,14 @@ async def report(self, result, name, event): class CSPExtractor(BaseExtractor): regexes = {"CSP": r"(?i)(?m)Content-Security-Policy:.+$"} - def extract_domains(self, csp): - domains = dns_name_regex.findall(csp) + async def extract_domains(self, csp): + domains = await self.helpers.re_findall(dns_name_regex, csp) unique_domains = set(domains) return unique_domains async def search(self, content, event, **kwargs): async for csp, name in self._search(content, event, **kwargs): - extracted_domains = self.extract_domains(csp) + extracted_domains = await self.extract_domains(csp) for domain in extracted_domains: await self.report(domain, event, **kwargs) @@ -125,7 +126,7 @@ async def _search(self, content, event, **kwargs): for name, regex in self.compiled_regexes.items(): # yield to event loop await self.excavate.helpers.sleep(0) - for result in regex.findall(content): + for result in await self.helpers.re_findall(regex, content): if name.startswith("full"): protocol, other = result result = f"{protocol}://{other}" diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py index d9462ae19..d94a3b0a2 100644 --- a/bbot/modules/secretsdb.py +++ b/bbot/modules/secretsdb.py @@ -46,7 +46,7 @@ async def setup(self): async def handle_event(self, event): resp_body = event.data.get("body", "") resp_headers = event.data.get("raw_header", "") - all_matches = await self.scan.run_in_executor(self.search_data, resp_body, resp_headers) + all_matches = await self.helpers.run_in_executor(self.search_data, resp_body, resp_headers) for matches, name in all_matches: matches = [m.string[m.start() : m.end()] for m in matches] description = f"Possible secret ({name}): {matches}" diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index c6fec1ea9..42f34d23e 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -119,7 +119,7 @@ async def visit_host(self, host, port): # Connect to the host try: transport, _ = await asyncio.wait_for( - self.scan._loop.create_connection(lambda: asyncio.Protocol(), host, port, ssl=ssl_context), + self.helpers.loop.create_connection(lambda: asyncio.Protocol(), host, port, ssl=ssl_context), timeout=self.timeout, ) except asyncio.TimeoutError: diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index 8fd0bc3d4..b8786e494 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -34,14 +34,14 @@ async def filter_event(self, event): async def handle_event(self, event): url = f"{event.parsed.scheme}://{event.parsed.netloc}/" - WW = await self.scan.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False) - waf_detections = await self.scan.run_in_executor(WW.identwaf) + WW = await self.helpers.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False) + waf_detections = await self.helpers.run_in_executor(WW.identwaf) if waf_detections: for waf in waf_detections: await self.emit_event({"host": str(event.host), "url": url, "WAF": waf}, "WAF", source=event) else: if self.config.get("generic_detect") == True: - generic = await self.scan.run_in_executor(WW.genericdetect) + generic = await self.helpers.run_in_executor(WW.genericdetect) if generic: await self.emit_event( { diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index 00e18f429..24fa54bcf 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -23,11 +23,11 @@ class wappalyzer(BaseModule): _max_event_handlers = 5 async def setup(self): - self.wappalyzer = await self.scan.run_in_executor(Wappalyzer.latest) + self.wappalyzer = await self.helpers.run_in_executor(Wappalyzer.latest) return True async def handle_event(self, event): - for res in await self.scan.run_in_executor(self.wappalyze, event.data): + for res in await self.helpers.run_in_executor(self.wappalyze, event.data): await self.emit_event( {"technology": res.lower(), "url": event.data["url"], "host": str(event.host)}, "TECHNOLOGY", event ) diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 92dc78db5..526e0b3eb 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -56,7 +56,7 @@ async def query(self, query): dns_names = set() collapsed_urls = 0 start_time = datetime.now() - parsed_urls = await self.scan.run_in_executor_mp( + parsed_urls = await self.helpers.run_in_executor_mp( self.helpers.validators.collapse_urls, urls, threshold=self.garbage_threshold, diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 36428cc53..c231b9a3b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -6,11 +6,8 @@ import contextlib from pathlib import Path from sys import exc_info -import multiprocessing as mp from datetime import datetime -from functools import partial from collections import OrderedDict -from concurrent.futures import ProcessPoolExecutor from bbot import __version__ @@ -207,16 +204,6 @@ def __init__( self.ticker_task = None self.dispatcher_tasks = [] - # multiprocessing thread pool - start_method = mp.get_start_method() - if start_method != "spawn": - self.warning(f"Multiprocessing spawn method is set to {start_method}.") - - # we spawn 1 fewer processes than cores - # this helps to avoid locking up the system or competing with the main python process for cpu time - num_processes = max(1, mp.cpu_count() - 1) - self.process_pool = ProcessPoolExecutor(max_workers=num_processes) - self._stopping = False self._dns_regexes = None @@ -758,7 +745,7 @@ def _cancel_tasks(self): tasks += self._manager_worker_loop_tasks self.helpers.cancel_tasks_sync(tasks) # process pool - self.process_pool.shutdown(cancel_futures=True) + self.helpers.process_pool.shutdown(cancel_futures=True) async def _report(self): """Asynchronously executes the `report()` method for each module in the scan. @@ -918,29 +905,6 @@ def root_event(self): root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") return root_event - def run_in_executor(self, callback, *args, **kwargs): - """ - Run a synchronous task in the event loop's default thread pool executor - - Examples: - Execute callback: - >>> result = await self.scan.run_in_executor(callback_fn, arg1, arg2) - """ - callback = partial(callback, **kwargs) - return self._loop.run_in_executor(None, callback, *args) - - def run_in_executor_mp(self, callback, *args, **kwargs): - """ - Same as run_in_executor() except with a process pool executor - Use only in cases where callback is CPU-bound - - Examples: - Execute callback: - >>> result = await self.scan.run_in_executor_mp(callback_fn, arg1, arg2) - """ - callback = partial(callback, **kwargs) - return self._loop.run_in_executor(self.process_pool, callback, *args) - @property def dns_regexes(self): """ @@ -1109,12 +1073,6 @@ def _fail_setup(self, msg): msg += " (--force to run module anyway)" raise ScanError(msg) - @property - def _loop(self): - if self.__loop is None: - self.__loop = asyncio.get_event_loop() - return self.__loop - def _load_modules(self, modules): modules = [str(m) for m in modules] loaded_modules = {} diff --git a/poetry.lock b/poetry.lock index 0a89ff3bc..74d992ebc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2613,4 +2613,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "daf56ec78cff336b530e48e83f0a854c3356a802e5e0cf9456b7a5adfe962354" +content-hash = "9665625c52f491373ac3f4306cacd3626e9c4dbc6ee79123c387693e2aa74ac7" diff --git a/pyproject.toml b/pyproject.toml index 4058cacb7..21ff710c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ cachetools = "^5.3.2" socksio = "^1.0.0" jinja2 = "^3.1.3" pyzmq = "^25.1.2" +regex = "^2024.4.16" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From 71e0c23b933f8fa038ff3bae33ed1c18f25c0997 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 Apr 2024 17:23:13 -0400 Subject: [PATCH 149/171] remove debug statement --- bbot/modules/internal/dns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index baaefc914..ea5e4efcf 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -179,7 +179,6 @@ async def handle_event(self, event, kwargs): source_event = self.scan.make_event(event.host, "DNS_NAME", module=self.host_module, source=event) # only emit the event if it's not already in the parent chain if source_event is not None and (source_event.always_emit or source_event not in event.get_sources()): - self.critical(f"SPECULATING {event.host} FROM {event}") source_event.scope_distance = event.scope_distance if "target" in event.tags: source_event.add_tag("target") From 8717ea49f604829f14c929828625d7f131659979 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 01:42:13 -0400 Subject: [PATCH 150/171] more WIP regex optimizations --- bbot/core/helpers/helper.py | 25 +---------- bbot/core/helpers/misc.py | 2 +- bbot/core/helpers/regex.py | 65 +++++++++++++++++++++++++++ bbot/core/helpers/regexes.py | 4 ++ bbot/modules/ajaxpro.py | 4 +- bbot/modules/azure_tenant.py | 6 +-- bbot/modules/dehashed.py | 2 +- bbot/modules/emailformat.py | 2 +- bbot/modules/hunt.py | 3 +- bbot/modules/internal/cloud.py | 2 +- bbot/modules/internal/excavate.py | 10 ++--- bbot/modules/massdns.py | 2 +- bbot/modules/oauth.py | 2 +- bbot/modules/paramminer_headers.py | 8 ++-- bbot/modules/pgp.py | 2 +- bbot/modules/report/asn.py | 2 +- bbot/modules/sitedossier.py | 11 +++-- bbot/modules/skymem.py | 12 +++-- bbot/scanner/scanner.py | 2 +- bbot/test/test_step_1/test_helpers.py | 22 +++++---- 20 files changed, 122 insertions(+), 66 deletions(-) create mode 100644 bbot/core/helpers/regex.py diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index ecf7c7cfc..16afc05cd 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -1,7 +1,6 @@ import os import asyncio import logging -import regex as re from pathlib import Path import multiprocessing as mp from functools import partial @@ -12,6 +11,7 @@ from .dns import DNSHelper from .web import WebHelper from .diff import HttpCompare +from .regex import RegexHelper from .wordcloud import WordCloud from .interactsh import Interactsh from ...scanner.target import Target @@ -84,34 +84,13 @@ def __init__(self, preset): self.cloud = cloud_providers + self.re = RegexHelper(self) self.dns = DNSHelper(self) self.web = WebHelper(self) self.depsinstaller = DepsInstaller(self) self.word_cloud = WordCloud(self) self.dummy_modules = {} - def ensure_compiled_regex(self, r): - """ - Make sure a regex has been compiled - """ - if not isinstance(r, re.Pattern): - raise ValueError("Regex must be compiled first!") - - async def re_search(self, compiled_regex, *args, **kwargs): - self.ensure_compiled_regex(compiled_regex) - return await self.run_in_executor(compiled_regex.search, *args, **kwargs) - - async def re_findall(self, compiled_regex, *args, **kwargs): - self.ensure_compiled_regex(compiled_regex) - return await self.run_in_executor(compiled_regex.findall, *args, **kwargs) - - async def re_finditer(self, compiled_regex, *args, **kwargs): - self.ensure_compiled_regex(compiled_regex) - return await self.run_in_executor(self._re_finditer, compiled_regex, *args, **kwargs) - - def _re_finditer(self, compiled_regex, *args, **kwargs): - return list(compiled_regex.finditer(*args, **kwargs)) - def interactsh(self, *args, **kwargs): return Interactsh(self, *args, **kwargs) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 7956466eb..27a3718ed 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1,5 +1,4 @@ import os -import re import sys import json import random @@ -7,6 +6,7 @@ import asyncio import logging import ipaddress +import regex as re import subprocess as sp from pathlib import Path from contextlib import suppress diff --git a/bbot/core/helpers/regex.py b/bbot/core/helpers/regex.py new file mode 100644 index 000000000..99116a5c8 --- /dev/null +++ b/bbot/core/helpers/regex.py @@ -0,0 +1,65 @@ +import regex as re +from . import misc + + +class RegexHelper: + """ + Class for misc CPU-intensive regex operations + + Offloads regex processing to other CPU cores via GIL release + thread pool + """ + + def __init__(self, parent_helper): + self.parent_helper = parent_helper + + def ensure_compiled_regex(self, r): + """ + Make sure a regex has been compiled + """ + if not isinstance(r, re.Pattern): + raise ValueError("Regex must be compiled first!") + + def compile(self, *args, **kwargs): + return re.compile(*args, **kwargs) + + async def search(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs) + + async def findall(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.parent_helper.run_in_executor(compiled_regex.findall, *args, **kwargs) + + async def finditer(self, compiled_regex, *args, **kwargs): + self.ensure_compiled_regex(compiled_regex) + return await self.parent_helper.run_in_executor(self._finditer, compiled_regex, *args, **kwargs) + + async def finditer_multi(self, compiled_regexes, *args, **kwargs): + for r in compiled_regexes: + self.ensure_compiled_regex(r) + return await self.parent_helper.run_in_executor(self._finditer_multi, compiled_regexes, *args, **kwargs) + + def _finditer_multi(self, compiled_regexes, *args, **kwargs): + matches = [] + for r in compiled_regexes: + for m in r.finditer(*args, **kwargs): + matches.append(m) + return matches + + def _finditer(self, compiled_regex, *args, **kwargs): + return list(compiled_regex.finditer(*args, **kwargs)) + + async def extract_params_html(self, *args, **kwargs): + return await self.parent_helper.run_in_executor(misc.extract_params_html, *args, **kwargs) + + async def extract_emails(self, *args, **kwargs): + return await self.parent_helper.run_in_executor(misc.extract_emails, *args, **kwargs) + + async def search_dict_values(self, *args, **kwargs): + def _search_dict_values(*_args, **_kwargs): + return list(misc.search_dict_values(*_args, **_kwargs)) + + return await self.parent_helper.run_in_executor(_search_dict_values, *args, **kwargs) + + async def recursive_decode(self, *args, **kwargs): + return await self.parent_helper.run_in_executor(misc.recursive_decode, *args, **kwargs) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index f5fb78f4c..4e2ada0c2 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -104,3 +104,7 @@ _extract_host_regex = r"(?:[a-z0-9]{1,20}://)?(?:[^?]*@)?(" + valid_netloc + ")" extract_host_regex = re.compile(_extract_host_regex, re.I) + +# for use in recursive_decode() +encoded_regex = re.compile(r"%[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\[ntrbv]") +backslash_regex = re.compile(r"(?P\\+)(?P[ntrvb])") diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py index 46d475cca..ba3e0eb3e 100644 --- a/bbot/modules/ajaxpro.py +++ b/bbot/modules/ajaxpro.py @@ -1,4 +1,4 @@ -import re +import regex as re from bbot.modules.base import BaseModule @@ -38,7 +38,7 @@ async def handle_event(self, event): elif event.type == "HTTP_RESPONSE": resp_body = event.data.get("body", None) if resp_body: - ajaxpro_regex_result = self.ajaxpro_regex.search(resp_body) + ajaxpro_regex_result = await self.helpers.re.search(self.ajaxpro_regex, resp_body) if ajaxpro_regex_result: ajax_pro_path = ajaxpro_regex_result.group(0) await self.emit_event( diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py index 909acbe20..a15bbb68f 100644 --- a/bbot/modules/azure_tenant.py +++ b/bbot/modules/azure_tenant.py @@ -1,4 +1,4 @@ -import re +import regex as re from contextlib import suppress from bbot.modules.base import BaseModule @@ -25,7 +25,7 @@ async def handle_event(self, event): tenant_id = None authorization_endpoint = openid_config.get("authorization_endpoint", "") - matches = self.helpers.regexes.uuid_regex.findall(authorization_endpoint) + matches = await self.helpers.re.findall(self.helpers.regexes.uuid_regex, authorization_endpoint) if matches: tenant_id = matches[0] @@ -86,7 +86,7 @@ async def query(self, domain): if status_code not in (200, 421): self.verbose(f'Error retrieving azure_tenant domains for "{domain}" (status code: {status_code})') return set(), dict() - found_domains = list(set(self.d_xml_regex.findall(r.text))) + found_domains = list(set(await self.helpers.re.findall(self.d_xml_regex, r.text))) domains = set() for d in found_domains: diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index c1a35c419..caa5fb662 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -33,7 +33,7 @@ async def handle_event(self, event): for entry in entries: # we have to clean up the email field because dehashed does a poor job of it email_str = entry.get("email", "").replace("\\", "") - found_emails = list(self.helpers.extract_emails(email_str)) + found_emails = list(await self.helpers.re.extract_emails(email_str)) if not found_emails: self.debug(f"Invalid email from dehashed.com: {email_str}") continue diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py index 000c3d5cf..31cff1468 100644 --- a/bbot/modules/emailformat.py +++ b/bbot/modules/emailformat.py @@ -17,6 +17,6 @@ async def handle_event(self, event): r = await self.request_with_fail_count(url) if not r: return - for email in self.helpers.extract_emails(r.text): + for email in await self.helpers.re.extract_emails(r.text): if email.endswith(query): await self.emit_event(email, "EMAIL_ADDRESS", source=event) diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index add45b665..dd591a345 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -1,7 +1,6 @@ # adapted from https://github.com/bugcrowd/HUNT from bbot.modules.base import BaseModule -from bbot.core.helpers.misc import extract_params_html hunt_param_dict = { "Command Injection": [ @@ -281,7 +280,7 @@ class hunt(BaseModule): async def handle_event(self, event): body = event.data.get("body", "") - for p in extract_params_html(body): + for p in await self.helpers.extract_params_html(body): for k in hunt_param_dict.keys(): if p.lower() in hunt_param_dict[k]: description = f"Found potential {k.upper()} parameter [{p}]" diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index 6bfceacff..14aaba930 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -43,7 +43,7 @@ async def handle_event(self, event, kwargs): for sig in sigs: matches = [] if event.type == "HTTP_RESPONSE": - matches = sig.findall(event.data.get("body", "")) + matches = await self.helpers.re.findall(sig, event.data.get("body", "")) elif event.type.startswith("DNS_NAME"): for host in hosts_to_check: match = sig.match(host) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 199a72b5e..6b819c0d2 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -30,7 +30,7 @@ async def _search(self, content, event, **kwargs): for name, regex in self.compiled_regexes.items(): # yield to event loop await self.excavate.helpers.sleep(0) - for result in await self.helpers.re_findall(regex, content): + for result in await self.helpers.re.findall(regex, content): yield result, name async def report(self, result, name, event): @@ -41,7 +41,7 @@ class CSPExtractor(BaseExtractor): regexes = {"CSP": r"(?i)(?m)Content-Security-Policy:.+$"} async def extract_domains(self, csp): - domains = await self.helpers.re_findall(dns_name_regex, csp) + domains = await self.helpers.re.findall(dns_name_regex, csp) unique_domains = set(domains) return unique_domains @@ -126,7 +126,7 @@ async def _search(self, content, event, **kwargs): for name, regex in self.compiled_regexes.items(): # yield to event loop await self.excavate.helpers.sleep(0) - for result in await self.helpers.re_findall(regex, content): + for result in await self.helpers.re.findall(regex, content): if name.startswith("full"): protocol, other = result result = f"{protocol}://{other}" @@ -387,7 +387,7 @@ async def handle_event(self, event): else: self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}") - body = self.helpers.recursive_decode(event.data.get("body", "")) + body = await self.helpers.re.recursive_decode(event.data.get("body", "")) await self.search( body, @@ -405,7 +405,7 @@ async def handle_event(self, event): consider_spider_danger=True, ) - headers = self.helpers.recursive_decode(event.data.get("raw_header", "")) + headers = await self.helpers.re.recursive_decode(event.data.get("raw_header", "")) await self.search( headers, [self.hostname, self.url, self.email, self.error_extractor, self.jwt, self.serialization, self.csp], diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 540b84041..ffacb8c64 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -1,7 +1,7 @@ -import re import json import random import subprocess +import regex as re from bbot.modules.templates.subdomain_enum import subdomain_enum diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 7e376000e..fd6188acd 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -119,7 +119,7 @@ async def getoidc(self, url): return url, token_endpoint, results if json and isinstance(json, dict): token_endpoint = json.get("token_endpoint", "") - for found in self.helpers.search_dict_values(json, *self.regexes): + for found in await self.helpers.re.search_dict_values(json, *self.regexes): results.add(found) results -= {token_endpoint} return url, token_endpoint, results diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 7d2174bbe..35b5c0df1 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -1,6 +1,6 @@ from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from bbot.core.helpers.misc import extract_params_json, extract_params_xml, extract_params_html +from bbot.core.helpers.misc import extract_params_json, extract_params_xml class paramminer_headers(BaseModule): @@ -158,7 +158,7 @@ async def handle_event(self, event): wl = set(self.wl) if self.config.get("http_extract"): - extracted_words = self.load_extracted_words(event.data.get("body"), event.data.get("content_type")) + extracted_words = await self.load_extracted_words(event.data.get("body"), event.data.get("content_type")) if extracted_words: self.debug(f"Extracted {str(len(extracted_words))} words from {url}") self.extracted_words_master.update(extracted_words - wl) @@ -195,7 +195,7 @@ def gen_count_args(self, url): yield header_count, (url,), {"headers": fake_headers} header_count -= 5 - def load_extracted_words(self, body, content_type): + async def load_extracted_words(self, body, content_type): if not body: return None if content_type and "json" in content_type.lower(): @@ -203,7 +203,7 @@ def load_extracted_words(self, body, content_type): elif content_type and "xml" in content_type.lower(): return extract_params_xml(body) else: - return set(extract_params_html(body)) + return set(await self.helpers.extract_params_html(body)) async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False): if reasons is None: diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index 2c378f585..78becbf0e 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -28,7 +28,7 @@ async def query(self, query): url = url.replace("", self.helpers.quote(query)) response = await self.helpers.request(url) if response is not None: - for email in self.helpers.extract_emails(response.text): + for email in await self.helpers.re.extract_emails(response.text): email = email.lower() if email.endswith(query): results.add(email) diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index f906c785e..982c76584 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -149,7 +149,7 @@ async def get_asn_metadata_ripe(self, asn_number): for item in record: key = item.get("key", "") value = item.get("value", "") - for email in self.helpers.extract_emails(value): + for email in await self.helpers.re.extract_emails(value): emails.add(email.lower()) if not key: continue diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py index 86872c052..e6571ea85 100644 --- a/bbot/modules/sitedossier.py +++ b/bbot/modules/sitedossier.py @@ -36,12 +36,11 @@ async def query(self, query, parse_fn=None, request_fn=None): if response.status_code == 302: self.verbose("Hit rate limit captcha") break - for regex in self.scan.dns_regexes: - for match in regex.finditer(response.text): - hostname = match.group().lower() - if hostname and hostname not in results: - results.add(hostname) - yield hostname + for match in await self.helpers.re.finditer_multi(self.scan.dns_regexes, response.text): + hostname = match.group().lower() + if hostname and hostname not in results: + results.add(hostname) + yield hostname if ' Date: Fri, 19 Apr 2024 09:30:01 -0400 Subject: [PATCH 151/171] steady work on regexes --- bbot/core/helpers/regex.py | 7 +++++++ bbot/test/test_step_1/test_helpers.py | 18 +++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/bbot/core/helpers/regex.py b/bbot/core/helpers/regex.py index 99116a5c8..f85fb72a5 100644 --- a/bbot/core/helpers/regex.py +++ b/bbot/core/helpers/regex.py @@ -7,6 +7,10 @@ class RegexHelper: Class for misc CPU-intensive regex operations Offloads regex processing to other CPU cores via GIL release + thread pool + + For quick, one-off regexes, you don't need to use this helper. + Only use this helper if you're searching large bodies of text + or if your regex is CPU-intensive """ def __init__(self, parent_helper): @@ -35,6 +39,9 @@ async def finditer(self, compiled_regex, *args, **kwargs): return await self.parent_helper.run_in_executor(self._finditer, compiled_regex, *args, **kwargs) async def finditer_multi(self, compiled_regexes, *args, **kwargs): + """ + Same as finditer() but with multiple regexes + """ for r in compiled_regexes: self.ensure_compiled_regex(r) return await self.parent_helper.run_in_executor(self._finditer_multi, compiled_regexes, *args, **kwargs) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 5d1350a0b..0ce3e0c76 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -360,25 +360,29 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): assert helpers.smart_encode_punycode("ドメイン.テスト:80") == "xn--eckwd4c7c.xn--zckzah:80" assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah:80") == "ドメイン.テスト:80" - assert await helpers.recursive_decode("Hello%20world%21") == "Hello world!" - assert await helpers.recursive_decode("Hello%20%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442") == "Hello Привет" + assert await helpers.re.recursive_decode("Hello%20world%21") == "Hello world!" assert ( - await helpers.recursive_decode("%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021") + await helpers.re.recursive_decode("Hello%20%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442") == "Hello Привет" + ) + assert ( + await helpers.re.recursive_decode("%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021") == " Привет!" ) - assert await helpers.recursive_decode("Hello%2520world%2521") == "Hello world!" + assert await helpers.re.recursive_decode("Hello%2520world%2521") == "Hello world!" assert ( - await helpers.recursive_decode("Hello%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442") + await helpers.re.recursive_decode( + "Hello%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442" + ) == "Hello Привет" ) assert ( - await helpers.recursive_decode( + await helpers.re.recursive_decode( "%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442%255Cu0021" ) == " Привет!" ) assert ( - await helpers.recursive_decode(r"Hello\\nWorld\\\tGreetings\\\\nMore\nText") + await helpers.re.recursive_decode(r"Hello\\nWorld\\\tGreetings\\\\nMore\nText") == "Hello\nWorld\tGreetings\nMore\nText" ) From 7899408a2a83e264b51dc1ec1c780b6b163f5274 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 11:29:44 -0400 Subject: [PATCH 152/171] don't start engine until necessary --- bbot/core/engine.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 06f965c45..c72eecbb3 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -8,7 +8,7 @@ import traceback import zmq.asyncio from pathlib import Path -from contextlib import contextmanager +from contextlib import asynccontextmanager, suppress from bbot.core import CORE from bbot.core.helpers.misc import rand_string @@ -31,12 +31,12 @@ def __init__(self, **kwargs): self.socket_address = f"zmq_{rand_string(8)}.sock" self.socket_path = Path(tempfile.gettempdir()) / self.socket_address self.server_kwargs = kwargs.pop("server_kwargs", {}) - self.server_process = self.start_server() + self._server_process = None self.context = zmq.asyncio.Context() atexit.register(self.cleanup) async def run_and_return(self, command, **kwargs): - with self.new_socket() as socket: + async with self.new_socket() as socket: message = self.make_message(command, args=kwargs) await socket.send(message) binary = await socket.recv() @@ -50,7 +50,7 @@ async def run_and_return(self, command, **kwargs): async def run_and_yield(self, command, **kwargs): message = self.make_message(command, args=kwargs) - with self.new_socket() as socket: + async with self.new_socket() as socket: await socket.send(message) while 1: binary = await socket.recv() @@ -86,7 +86,7 @@ def make_message(self, command, args): def available_commands(self): return [s for s in self.CMDS if isinstance(s, str)] - def start_server(self, **server_kwargs): + def start_server(self): process = CORE.create_process( target=self.server_process, args=( @@ -100,17 +100,30 @@ def start_server(self, **server_kwargs): @staticmethod def server_process(server_class, socket_path, **kwargs): - engine_server = server_class(socket_path, **kwargs) - asyncio.run(engine_server.worker()) - - @contextmanager - def new_socket(self): + try: + engine_server = server_class(socket_path, **kwargs) + asyncio.run(engine_server.worker()) + except (asyncio.CancelledError, KeyboardInterrupt): + pass + except Exception: + import traceback + + log = logging.getLogger("bbot.core.engine.server") + log.critical(f"Unhandled error in {server_class.__name__} server process: {traceback.format_exc()}") + + @asynccontextmanager + async def new_socket(self): + if self._server_process is None: + self._server_process = self.start_server() + while not self.socket_path.exists(): + await asyncio.sleep(0.1) socket = self.context.socket(zmq.DEALER) socket.connect(f"ipc://{self.socket_path}") try: yield socket finally: - socket.close() + with suppress(Exception): + socket.close() def cleanup(self): # delete socket file on exit @@ -158,7 +171,6 @@ async def worker(self): try: while 1: client_id, binary = await self.socket.recv_multipart() - # self.log.debug(f"{self.name} got binary: {binary}") message = pickle.loads(binary) self.log.debug(f"{self.name} got message: {message}") @@ -189,4 +201,5 @@ async def worker(self): self.log.error(f"Error in EngineServer worker: {e}") self.log.trace(traceback.format_exc()) finally: - self.socket.close() + with suppress(Exception): + self.socket.close() From 46a8d1c8aacf89d189c2a9b72139feff996b20b3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 11:40:30 -0400 Subject: [PATCH 153/171] re-add poetry.lock --- poetry.lock | 2522 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2522 insertions(+) create mode 100644 poetry.lock diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000..966b5e9c9 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,2522 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "ansible" +version = "8.7.0" +description = "Radically simple IT automation" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ansible-8.7.0-py3-none-any.whl", hash = "sha256:fa7d3bc2dfdb0ab031df645814ff86b15cb5ec041bfbee4041f795abfa5646ca"}, + {file = "ansible-8.7.0.tar.gz", hash = "sha256:3a5ca5152e4547d590e40b542d76b18dbbe2b36da4edd00a13a7c51a374ff737"}, +] + +[package.dependencies] +ansible-core = ">=2.15.7,<2.16.0" + +[[package]] +name = "ansible-core" +version = "2.15.10" +description = "Radically simple IT automation" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ansible-core-2.15.10.tar.gz", hash = "sha256:954dbe8e4e802a4dd5df0366193975b692a05806aa8d7358418a7e617346b20f"}, + {file = "ansible_core-2.15.10-py3-none-any.whl", hash = "sha256:42e49f1a6d8cf6cccde775c06c1394885353b71ad9e5f670c6f32d2890127ce8"}, +] + +[package.dependencies] +cryptography = "*" +importlib-resources = {version = ">=5.0,<5.1", markers = "python_version < \"3.10\""} +jinja2 = ">=3.0.0" +packaging = "*" +PyYAML = ">=5.1" +resolvelib = ">=0.5.3,<1.1.0" + +[[package]] +name = "ansible-runner" +version = "2.3.6" +description = "\"Consistent Ansible Python API and CLI with container and process isolation runtime capabilities\"" +optional = false +python-versions = ">=3.8" +files = [ + {file = "ansible-runner-2.3.6.tar.gz", hash = "sha256:b2174a12dcad2dc2f342ea82876898f568a0b66c53568600bf80577158fcba1c"}, + {file = "ansible_runner-2.3.6-py3-none-any.whl", hash = "sha256:4f153d9c3000a61b82d7253ca292849e3ad2c5d68dfff4377a6b98c4e6ff6c3e"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=4.6,<6.3", markers = "python_version < \"3.10\""} +packaging = "*" +pexpect = ">=4.5" +python-daemon = "*" +pyyaml = "*" +six = "*" + +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +optional = false +python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] + +[[package]] +name = "anyio" +version = "4.3.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, + {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "babel" +version = "2.14.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, + {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, +] + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "black" +version = "24.4.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436"}, + {file = "black-24.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf"}, + {file = "black-24.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad"}, + {file = "black-24.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb"}, + {file = "black-24.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8"}, + {file = "black-24.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745"}, + {file = "black-24.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070"}, + {file = "black-24.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397"}, + {file = "black-24.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2"}, + {file = "black-24.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33"}, + {file = "black-24.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965"}, + {file = "black-24.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd"}, + {file = "black-24.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1"}, + {file = "black-24.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8"}, + {file = "black-24.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d"}, + {file = "black-24.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3"}, + {file = "black-24.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665"}, + {file = "black-24.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6"}, + {file = "black-24.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e"}, + {file = "black-24.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702"}, + {file = "black-24.4.0-py3-none-any.whl", hash = "sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e"}, + {file = "black-24.4.0.tar.gz", hash = "sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "cachetools" +version = "5.3.3" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, +] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "cloudcheck" +version = "3.1.0.318" +description = "Check whether an IP address belongs to a cloud provider" +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "cloudcheck-3.1.0.318-py3-none-any.whl", hash = "sha256:471dba97531e1f60aadab8daa6cb1d63727f67c16fd7b4758db46c9af2f362f1"}, + {file = "cloudcheck-3.1.0.318.tar.gz", hash = "sha256:ba7fcc026817aa05f74c7789d2ac306469f3143f91b3ea9f95c57c70a7b0b787"}, +] + +[package.dependencies] +httpx = ">=0.26,<0.28" +pydantic = ">=2.4.2,<3.0.0" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.4.4" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2"}, + {file = "coverage-7.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf"}, + {file = "coverage-7.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8"}, + {file = "coverage-7.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562"}, + {file = "coverage-7.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2"}, + {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7"}, + {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87"}, + {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c"}, + {file = "coverage-7.4.4-cp310-cp310-win32.whl", hash = "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d"}, + {file = "coverage-7.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f"}, + {file = "coverage-7.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf"}, + {file = "coverage-7.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083"}, + {file = "coverage-7.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63"}, + {file = "coverage-7.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f"}, + {file = "coverage-7.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227"}, + {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd"}, + {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384"}, + {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b"}, + {file = "coverage-7.4.4-cp311-cp311-win32.whl", hash = "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286"}, + {file = "coverage-7.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec"}, + {file = "coverage-7.4.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76"}, + {file = "coverage-7.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818"}, + {file = "coverage-7.4.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978"}, + {file = "coverage-7.4.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70"}, + {file = "coverage-7.4.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51"}, + {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c"}, + {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48"}, + {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9"}, + {file = "coverage-7.4.4-cp312-cp312-win32.whl", hash = "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0"}, + {file = "coverage-7.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e"}, + {file = "coverage-7.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384"}, + {file = "coverage-7.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1"}, + {file = "coverage-7.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a"}, + {file = "coverage-7.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409"}, + {file = "coverage-7.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e"}, + {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd"}, + {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7"}, + {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c"}, + {file = "coverage-7.4.4-cp38-cp38-win32.whl", hash = "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e"}, + {file = "coverage-7.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8"}, + {file = "coverage-7.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d"}, + {file = "coverage-7.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357"}, + {file = "coverage-7.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e"}, + {file = "coverage-7.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e"}, + {file = "coverage-7.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4"}, + {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec"}, + {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd"}, + {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade"}, + {file = "coverage-7.4.4-cp39-cp39-win32.whl", hash = "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57"}, + {file = "coverage-7.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c"}, + {file = "coverage-7.4.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677"}, + {file = "coverage-7.4.4.tar.gz", hash = "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "42.0.5" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da"}, + {file = "cryptography-42.0.5-cp37-abi3-win32.whl", hash = "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74"}, + {file = "cryptography-42.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940"}, + {file = "cryptography-42.0.5-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30"}, + {file = "cryptography-42.0.5-cp39-abi3-win32.whl", hash = "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413"}, + {file = "cryptography-42.0.5-cp39-abi3-win_amd64.whl", hash = "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd"}, + {file = "cryptography-42.0.5.tar.gz", hash = "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deepdiff" +version = "7.0.1" +description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." +optional = false +python-versions = ">=3.8" +files = [ + {file = "deepdiff-7.0.1-py3-none-any.whl", hash = "sha256:447760081918216aa4fd4ca78a4b6a848b81307b2ea94c810255334b759e1dc3"}, + {file = "deepdiff-7.0.1.tar.gz", hash = "sha256:260c16f052d4badbf60351b4f77e8390bee03a0b516246f6839bc813fb429ddf"}, +] + +[package.dependencies] +ordered-set = ">=4.1.0,<4.2.0" + +[package.extras] +cli = ["click (==8.1.7)", "pyyaml (==6.0.1)"] +optimize = ["orjson"] + +[[package]] +name = "distlib" +version = "0.3.8" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, +] + +[[package]] +name = "dnspython" +version = "2.6.1" +description = "DNS toolkit" +optional = false +python-versions = ">=3.8" +files = [ + {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, + {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, +] + +[package.extras] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] +dnssec = ["cryptography (>=41)"] +doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] +doq = ["aioquic (>=0.9.25)"] +idna = ["idna (>=3.6)"] +trio = ["trio (>=0.23)"] +wmi = ["wmi (>=1.5.1)"] + +[[package]] +name = "docutils" +version = "0.21.1" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.9" +files = [ + {file = "docutils-0.21.1-py3-none-any.whl", hash = "sha256:14c8d34a55b46c88f9f714adb29cefbdd69fb82f3fef825e59c5faab935390d8"}, + {file = "docutils-0.21.1.tar.gz", hash = "sha256:65249d8a5345bc95e0f40f280ba63c98eb24de35c6c8f5b662e3e8948adea83f"}, +] + +[[package]] +name = "dunamai" +version = "1.20.0" +description = "Dynamic version generation" +optional = false +python-versions = ">=3.5" +files = [ + {file = "dunamai-1.20.0-py3-none-any.whl", hash = "sha256:a2185c227351a52a013c7d7a695d3f3cb6625c3eed14a5295adbbcc7e2f7f8d4"}, + {file = "dunamai-1.20.0.tar.gz", hash = "sha256:c3f1ee64a1e6cc9ebc98adafa944efaccd0db32482d2177e59c1ff6bdf23cd70"}, +] + +[package.dependencies] +packaging = ">=20.9" + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.13.4" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"}, + {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "flake8" +version = "7.0.0" +description = "the modular source code checker: pep8 pyflakes and co" +optional = false +python-versions = ">=3.8.1" +files = [ + {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, + {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, +] + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.11.0,<2.12.0" +pyflakes = ">=3.2.0,<3.3.0" + +[[package]] +name = "ghp-import" +version = "2.1.0" +description = "Copy your docs directly to the gh-pages branch." +optional = false +python-versions = "*" +files = [ + {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, + {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, +] + +[package.dependencies] +python-dateutil = ">=2.8.1" + +[package.extras] +dev = ["flake8", "markdown", "twine", "wheel"] + +[[package]] +name = "griffe" +version = "0.44.0" +description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "griffe-0.44.0-py3-none-any.whl", hash = "sha256:8a4471c469ba980b87c843f1168850ce39d0c1d0c7be140dca2480f76c8e5446"}, + {file = "griffe-0.44.0.tar.gz", hash = "sha256:34aee1571042f9bf00529bc715de4516fb6f482b164e90d030300601009e0223"}, +] + +[package.dependencies] +colorama = ">=0.4" + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.5" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.26.0)"] + +[[package]] +name = "httpx" +version = "0.26.0" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "identify" +version = "2.5.35" +description = "File identification library for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "identify-2.5.35-py2.py3-none-any.whl", hash = "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"}, + {file = "identify-2.5.35.tar.gz", hash = "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791"}, +] + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "importlib-metadata" +version = "6.2.1" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.2.1-py3-none-any.whl", hash = "sha256:f65e478a7c2177bd19517a3a15dac094d253446d8690c5f3e71e735a04312374"}, + {file = "importlib_metadata-6.2.1.tar.gz", hash = "sha256:5a66966b39ff1c14ef5b2d60c1d842b0141fefff0f4cc6365b4bc9446c652807"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "importlib-resources" +version = "5.0.7" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.6" +files = [ + {file = "importlib_resources-5.0.7-py3-none-any.whl", hash = "sha256:2238159eb743bd85304a16e0536048b3e991c531d1cd51c4a834d1ccf2829057"}, + {file = "importlib_resources-5.0.7.tar.gz", hash = "sha256:4df460394562b4581bb4e4087ad9447bd433148fba44241754ec3152499f1d1b"}, +] + +[package.extras] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-enabler", "pytest-flake8", "pytest-mypy"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "libsass" +version = "0.23.0" +description = "Sass for Python: A straightforward binding of libsass for Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "libsass-0.23.0-cp38-abi3-macosx_11_0_x86_64.whl", hash = "sha256:34cae047cbbfc4ffa832a61cbb110f3c95f5471c6170c842d3fed161e40814dc"}, + {file = "libsass-0.23.0-cp38-abi3-macosx_14_0_arm64.whl", hash = "sha256:ea97d1b45cdc2fc3590cb9d7b60f1d8915d3ce17a98c1f2d4dd47ee0d9c68ce6"}, + {file = "libsass-0.23.0-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4a218406d605f325d234e4678bd57126a66a88841cb95bee2caeafdc6f138306"}, + {file = "libsass-0.23.0-cp38-abi3-win32.whl", hash = "sha256:31e86d92a5c7a551df844b72d83fc2b5e50abc6fbbb31e296f7bebd6489ed1b4"}, + {file = "libsass-0.23.0-cp38-abi3-win_amd64.whl", hash = "sha256:a2ec85d819f353cbe807432d7275d653710d12b08ec7ef61c124a580a8352f3c"}, + {file = "libsass-0.23.0.tar.gz", hash = "sha256:6f209955ede26684e76912caf329f4ccb57e4a043fd77fe0e7348dd9574f1880"}, +] + +[[package]] +name = "livereload" +version = "2.6.3" +description = "Python LiveReload is an awesome tool for web developers" +optional = false +python-versions = "*" +files = [ + {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, + {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, +] + +[package.dependencies] +six = "*" +tornado = {version = "*", markers = "python_version > \"2.7\""} + +[[package]] +name = "lockfile" +version = "0.12.2" +description = "Platform-independent file locking module" +optional = false +python-versions = "*" +files = [ + {file = "lockfile-0.12.2-py2.py3-none-any.whl", hash = "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"}, + {file = "lockfile-0.12.2.tar.gz", hash = "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799"}, +] + +[[package]] +name = "lxml" +version = "5.2.1" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1f7785f4f789fdb522729ae465adcaa099e2a3441519df750ebdccc481d961a1"}, + {file = "lxml-5.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6cc6ee342fb7fa2471bd9b6d6fdfc78925a697bf5c2bcd0a302e98b0d35bfad3"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:794f04eec78f1d0e35d9e0c36cbbb22e42d370dda1609fb03bcd7aeb458c6377"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817d420c60a5183953c783b0547d9eb43b7b344a2c46f69513d5952a78cddf3"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2213afee476546a7f37c7a9b4ad4d74b1e112a6fafffc9185d6d21f043128c81"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b070bbe8d3f0f6147689bed981d19bbb33070225373338df755a46893528104a"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e02c5175f63effbd7c5e590399c118d5db6183bbfe8e0d118bdb5c2d1b48d937"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3dc773b2861b37b41a6136e0b72a1a44689a9c4c101e0cddb6b854016acc0aa8"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:d7520db34088c96cc0e0a3ad51a4fd5b401f279ee112aa2b7f8f976d8582606d"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:bcbf4af004f98793a95355980764b3d80d47117678118a44a80b721c9913436a"}, + {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a2b44bec7adf3e9305ce6cbfa47a4395667e744097faed97abb4728748ba7d47"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1c5bb205e9212d0ebddf946bc07e73fa245c864a5f90f341d11ce7b0b854475d"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2c9d147f754b1b0e723e6afb7ba1566ecb162fe4ea657f53d2139bbf894d050a"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3545039fa4779be2df51d6395e91a810f57122290864918b172d5dc7ca5bb433"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a91481dbcddf1736c98a80b122afa0f7296eeb80b72344d7f45dc9f781551f56"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2ddfe41ddc81f29a4c44c8ce239eda5ade4e7fc305fb7311759dd6229a080052"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a7baf9ffc238e4bf401299f50e971a45bfcc10a785522541a6e3179c83eabf0a"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:31e9a882013c2f6bd2f2c974241bf4ba68c85eba943648ce88936d23209a2e01"}, + {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0a15438253b34e6362b2dc41475e7f80de76320f335e70c5528b7148cac253a1"}, + {file = "lxml-5.2.1-cp310-cp310-win32.whl", hash = "sha256:6992030d43b916407c9aa52e9673612ff39a575523c5f4cf72cdef75365709a5"}, + {file = "lxml-5.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:da052e7962ea2d5e5ef5bc0355d55007407087392cf465b7ad84ce5f3e25fe0f"}, + {file = "lxml-5.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:70ac664a48aa64e5e635ae5566f5227f2ab7f66a3990d67566d9907edcbbf867"}, + {file = "lxml-5.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1ae67b4e737cddc96c99461d2f75d218bdf7a0c3d3ad5604d1f5e7464a2f9ffe"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f18a5a84e16886898e51ab4b1d43acb3083c39b14c8caeb3589aabff0ee0b270"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6f2c8372b98208ce609c9e1d707f6918cc118fea4e2c754c9f0812c04ca116d"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:394ed3924d7a01b5bd9a0d9d946136e1c2f7b3dc337196d99e61740ed4bc6fe1"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d077bc40a1fe984e1a9931e801e42959a1e6598edc8a3223b061d30fbd26bbc"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:764b521b75701f60683500d8621841bec41a65eb739b8466000c6fdbc256c240"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3a6b45da02336895da82b9d472cd274b22dc27a5cea1d4b793874eead23dd14f"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:5ea7b6766ac2dfe4bcac8b8595107665a18ef01f8c8343f00710b85096d1b53a"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e196a4ff48310ba62e53a8e0f97ca2bca83cdd2fe2934d8b5cb0df0a841b193a"}, + {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:200e63525948e325d6a13a76ba2911f927ad399ef64f57898cf7c74e69b71095"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dae0ed02f6b075426accbf6b2863c3d0a7eacc1b41fb40f2251d931e50188dad"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:ab31a88a651039a07a3ae327d68ebdd8bc589b16938c09ef3f32a4b809dc96ef"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df2e6f546c4df14bc81f9498bbc007fbb87669f1bb707c6138878c46b06f6510"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5dd1537e7cc06efd81371f5d1a992bd5ab156b2b4f88834ca852de4a8ea523fa"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9b9ec9c9978b708d488bec36b9e4c94d88fd12ccac3e62134a9d17ddba910ea9"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8e77c69d5892cb5ba71703c4057091e31ccf534bd7f129307a4d084d90d014b8"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a8d5c70e04aac1eda5c829a26d1f75c6e5286c74743133d9f742cda8e53b9c2f"}, + {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c94e75445b00319c1fad60f3c98b09cd63fe1134a8a953dcd48989ef42318534"}, + {file = "lxml-5.2.1-cp311-cp311-win32.whl", hash = "sha256:4951e4f7a5680a2db62f7f4ab2f84617674d36d2d76a729b9a8be4b59b3659be"}, + {file = "lxml-5.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:5c670c0406bdc845b474b680b9a5456c561c65cf366f8db5a60154088c92d102"}, + {file = "lxml-5.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:abc25c3cab9ec7fcd299b9bcb3b8d4a1231877e425c650fa1c7576c5107ab851"}, + {file = "lxml-5.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6935bbf153f9a965f1e07c2649c0849d29832487c52bb4a5c5066031d8b44fd5"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d793bebb202a6000390a5390078e945bbb49855c29c7e4d56a85901326c3b5d9"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd5562927cdef7c4f5550374acbc117fd4ecc05b5007bdfa57cc5355864e0a4"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0e7259016bc4345a31af861fdce942b77c99049d6c2107ca07dc2bba2435c1d9"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:530e7c04f72002d2f334d5257c8a51bf409db0316feee7c87e4385043be136af"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59689a75ba8d7ffca577aefd017d08d659d86ad4585ccc73e43edbfc7476781a"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f9737bf36262046213a28e789cc82d82c6ef19c85a0cf05e75c670a33342ac2c"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:3a74c4f27167cb95c1d4af1c0b59e88b7f3e0182138db2501c353555f7ec57f4"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:68a2610dbe138fa8c5826b3f6d98a7cfc29707b850ddcc3e21910a6fe51f6ca0"}, + {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f0a1bc63a465b6d72569a9bba9f2ef0334c4e03958e043da1920299100bc7c08"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c2d35a1d047efd68027817b32ab1586c1169e60ca02c65d428ae815b593e65d4"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:79bd05260359170f78b181b59ce871673ed01ba048deef4bf49a36ab3e72e80b"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:865bad62df277c04beed9478fe665b9ef63eb28fe026d5dedcb89b537d2e2ea6"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:44f6c7caff88d988db017b9b0e4ab04934f11e3e72d478031efc7edcac6c622f"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71e97313406ccf55d32cc98a533ee05c61e15d11b99215b237346171c179c0b0"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:057cdc6b86ab732cf361f8b4d8af87cf195a1f6dc5b0ff3de2dced242c2015e0"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f3bbbc998d42f8e561f347e798b85513ba4da324c2b3f9b7969e9c45b10f6169"}, + {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:491755202eb21a5e350dae00c6d9a17247769c64dcf62d8c788b5c135e179dc4"}, + {file = "lxml-5.2.1-cp312-cp312-win32.whl", hash = "sha256:8de8f9d6caa7f25b204fc861718815d41cbcf27ee8f028c89c882a0cf4ae4134"}, + {file = "lxml-5.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:f2a9efc53d5b714b8df2b4b3e992accf8ce5bbdfe544d74d5c6766c9e1146a3a"}, + {file = "lxml-5.2.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:70a9768e1b9d79edca17890175ba915654ee1725975d69ab64813dd785a2bd5c"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:5d5792e9b3fb8d16a19f46aa8208987cfeafe082363ee2745ea8b643d9cc5b45"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:88e22fc0a6684337d25c994381ed8a1580a6f5ebebd5ad41f89f663ff4ec2885"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:21c2e6b09565ba5b45ae161b438e033a86ad1736b8c838c766146eff8ceffff9"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_s390x.whl", hash = "sha256:afbbdb120d1e78d2ba8064a68058001b871154cc57787031b645c9142b937a62"}, + {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:627402ad8dea044dde2eccde4370560a2b750ef894c9578e1d4f8ffd54000461"}, + {file = "lxml-5.2.1-cp36-cp36m-win32.whl", hash = "sha256:e89580a581bf478d8dcb97d9cd011d567768e8bc4095f8557b21c4d4c5fea7d0"}, + {file = "lxml-5.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59565f10607c244bc4c05c0c5fa0c190c990996e0c719d05deec7030c2aa8289"}, + {file = "lxml-5.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:857500f88b17a6479202ff5fe5f580fc3404922cd02ab3716197adf1ef628029"}, + {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56c22432809085b3f3ae04e6e7bdd36883d7258fcd90e53ba7b2e463efc7a6af"}, + {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a55ee573116ba208932e2d1a037cc4b10d2c1cb264ced2184d00b18ce585b2c0"}, + {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:6cf58416653c5901e12624e4013708b6e11142956e7f35e7a83f1ab02f3fe456"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:64c2baa7774bc22dd4474248ba16fe1a7f611c13ac6123408694d4cc93d66dbd"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:74b28c6334cca4dd704e8004cba1955af0b778cf449142e581e404bd211fb619"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7221d49259aa1e5a8f00d3d28b1e0b76031655ca74bb287123ef56c3db92f213"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3dbe858ee582cbb2c6294dc85f55b5f19c918c2597855e950f34b660f1a5ede6"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:04ab5415bf6c86e0518d57240a96c4d1fcfc3cb370bb2ac2a732b67f579e5a04"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:6ab833e4735a7e5533711a6ea2df26459b96f9eec36d23f74cafe03631647c41"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f443cdef978430887ed55112b491f670bba6462cea7a7742ff8f14b7abb98d75"}, + {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"}, + {file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"}, + {file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"}, + {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5"}, + {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"}, + {file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"}, + {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"}, + {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3d30321949861404323c50aebeb1943461a67cd51d4200ab02babc58bd06a86"}, + {file = "lxml-5.2.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:b560e3aa4b1d49e0e6c847d72665384db35b2f5d45f8e6a5c0072e0283430533"}, + {file = "lxml-5.2.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:058a1308914f20784c9f4674036527e7c04f7be6fb60f5d61353545aa7fcb739"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:adfb84ca6b87e06bc6b146dc7da7623395db1e31621c4785ad0658c5028b37d7"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:417d14450f06d51f363e41cace6488519038f940676ce9664b34ebf5653433a5"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a2dfe7e2473f9b59496247aad6e23b405ddf2e12ef0765677b0081c02d6c2c0b"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bf2e2458345d9bffb0d9ec16557d8858c9c88d2d11fed53998512504cd9df49b"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:58278b29cb89f3e43ff3e0c756abbd1518f3ee6adad9e35b51fb101c1c1daaec"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:64641a6068a16201366476731301441ce93457eb8452056f570133a6ceb15fca"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:78bfa756eab503673991bdcf464917ef7845a964903d3302c5f68417ecdc948c"}, + {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:11a04306fcba10cd9637e669fd73aa274c1c09ca64af79c041aa820ea992b637"}, + {file = "lxml-5.2.1-cp38-cp38-win32.whl", hash = "sha256:66bc5eb8a323ed9894f8fa0ee6cb3e3fb2403d99aee635078fd19a8bc7a5a5da"}, + {file = "lxml-5.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:9676bfc686fa6a3fa10cd4ae6b76cae8be26eb5ec6811d2a325636c460da1806"}, + {file = "lxml-5.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cf22b41fdae514ee2f1691b6c3cdeae666d8b7fa9434de445f12bbeee0cf48dd"}, + {file = "lxml-5.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec42088248c596dbd61d4ae8a5b004f97a4d91a9fd286f632e42e60b706718d7"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd53553ddad4a9c2f1f022756ae64abe16da1feb497edf4d9f87f99ec7cf86bd"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feaa45c0eae424d3e90d78823f3828e7dc42a42f21ed420db98da2c4ecf0a2cb"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddc678fb4c7e30cf830a2b5a8d869538bc55b28d6c68544d09c7d0d8f17694dc"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:853e074d4931dbcba7480d4dcab23d5c56bd9607f92825ab80ee2bd916edea53"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc4691d60512798304acb9207987e7b2b7c44627ea88b9d77489bbe3e6cc3bd4"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:beb72935a941965c52990f3a32d7f07ce869fe21c6af8b34bf6a277b33a345d3"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:6588c459c5627fefa30139be4d2e28a2c2a1d0d1c265aad2ba1935a7863a4913"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:588008b8497667f1ddca7c99f2f85ce8511f8f7871b4a06ceede68ab62dff64b"}, + {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6787b643356111dfd4032b5bffe26d2f8331556ecb79e15dacb9275da02866e"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7c17b64b0a6ef4e5affae6a3724010a7a66bda48a62cfe0674dabd46642e8b54"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:27aa20d45c2e0b8cd05da6d4759649170e8dfc4f4e5ef33a34d06f2d79075d57"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d4f2cc7060dc3646632d7f15fe68e2fa98f58e35dd5666cd525f3b35d3fed7f8"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff46d772d5f6f73564979cd77a4fffe55c916a05f3cb70e7c9c0590059fb29ef"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:96323338e6c14e958d775700ec8a88346014a85e5de73ac7967db0367582049b"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:52421b41ac99e9d91934e4d0d0fe7da9f02bfa7536bb4431b4c05c906c8c6919"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7a7efd5b6d3e30d81ec68ab8a88252d7c7c6f13aaa875009fe3097eb4e30b84c"}, + {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ed777c1e8c99b63037b91f9d73a6aad20fd035d77ac84afcc205225f8f41188"}, + {file = "lxml-5.2.1-cp39-cp39-win32.whl", hash = "sha256:644df54d729ef810dcd0f7732e50e5ad1bd0a135278ed8d6bcb06f33b6b6f708"}, + {file = "lxml-5.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:9ca66b8e90daca431b7ca1408cae085d025326570e57749695d6a01454790e95"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9b0ff53900566bc6325ecde9181d89afadc59c5ffa39bddf084aaedfe3b06a11"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd6037392f2d57793ab98d9e26798f44b8b4da2f2464388588f48ac52c489ea1"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b9c07e7a45bb64e21df4b6aa623cb8ba214dfb47d2027d90eac197329bb5e94"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3249cc2989d9090eeac5467e50e9ec2d40704fea9ab72f36b034ea34ee65ca98"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f42038016852ae51b4088b2862126535cc4fc85802bfe30dea3500fdfaf1864e"}, + {file = "lxml-5.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:533658f8fbf056b70e434dff7e7aa611bcacb33e01f75de7f821810e48d1bb66"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:622020d4521e22fb371e15f580d153134bfb68d6a429d1342a25f051ec72df1c"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa7b51824aa0ee957ccd5a741c73e6851de55f40d807f08069eb4c5a26b2baa"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c6ad0fbf105f6bcc9300c00010a2ffa44ea6f555df1a2ad95c88f5656104817"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e233db59c8f76630c512ab4a4daf5a5986da5c3d5b44b8e9fc742f2a24dbd460"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6a014510830df1475176466b6087fc0c08b47a36714823e58d8b8d7709132a96"}, + {file = "lxml-5.2.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d38c8f50ecf57f0463399569aa388b232cf1a2ffb8f0a9a5412d0db57e054860"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5aea8212fb823e006b995c4dda533edcf98a893d941f173f6c9506126188860d"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff097ae562e637409b429a7ac958a20aab237a0378c42dabaa1e3abf2f896e5f"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f5d65c39f16717a47c36c756af0fb36144069c4718824b7533f803ecdf91138"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3d0c3dd24bb4605439bf91068598d00c6370684f8de4a67c2992683f6c309d6b"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e32be23d538753a8adb6c85bd539f5fd3b15cb987404327c569dfc5fd8366e85"}, + {file = "lxml-5.2.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cc518cea79fd1e2f6c90baafa28906d4309d24f3a63e801d855e7424c5b34144"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a0af35bd8ebf84888373630f73f24e86bf016642fb8576fba49d3d6b560b7cbc"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8aca2e3a72f37bfc7b14ba96d4056244001ddcc18382bd0daa087fd2e68a354"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ca1e8188b26a819387b29c3895c47a5e618708fe6f787f3b1a471de2c4a94d9"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c8ba129e6d3b0136a0f50345b2cb3db53f6bda5dd8c7f5d83fbccba97fb5dcb5"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e998e304036198b4f6914e6a1e2b6f925208a20e2042563d9734881150c6c246"}, + {file = "lxml-5.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d3be9b2076112e51b323bdf6d5a7f8a798de55fb8d95fcb64bd179460cdc0704"}, + {file = "lxml-5.2.1.tar.gz", hash = "sha256:3f7765e69bbce0906a7c74d5fe46d2c7a7596147318dbc08e4a2431f3060e306"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.10)"] + +[[package]] +name = "markdown" +version = "3.6" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, + {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +testing = ["coverage", "pyyaml"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + +[[package]] +name = "mergedeep" +version = "1.3.4" +description = "A deep merge function for 🐍." +optional = false +python-versions = ">=3.6" +files = [ + {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, + {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, +] + +[[package]] +name = "mkdocs" +version = "1.5.3" +description = "Project documentation with Markdown." +optional = false +python-versions = ">=3.7" +files = [ + {file = "mkdocs-1.5.3-py3-none-any.whl", hash = "sha256:3b3a78e736b31158d64dbb2f8ba29bd46a379d0c6e324c2246c3bc3d2189cfc1"}, + {file = "mkdocs-1.5.3.tar.gz", hash = "sha256:eb7c99214dcb945313ba30426c2451b735992c73c2e10838f76d09e39ff4d0e2"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""} +ghp-import = ">=1.0" +importlib-metadata = {version = ">=4.3", markers = "python_version < \"3.10\""} +jinja2 = ">=2.11.1" +markdown = ">=3.2.1" +markupsafe = ">=2.0.1" +mergedeep = ">=1.3.4" +packaging = ">=20.5" +pathspec = ">=0.11.1" +platformdirs = ">=2.2.0" +pyyaml = ">=5.1" +pyyaml-env-tag = ">=0.1" +watchdog = ">=2.0" + +[package.extras] +i18n = ["babel (>=2.9.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.3)", "jinja2 (==2.11.1)", "markdown (==3.2.1)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "packaging (==20.5)", "pathspec (==0.11.1)", "platformdirs (==2.2.0)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "typing-extensions (==3.10)", "watchdog (==2.0)"] + +[[package]] +name = "mkdocs-autorefs" +version = "1.0.1" +description = "Automatically link across pages in MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocs_autorefs-1.0.1-py3-none-any.whl", hash = "sha256:aacdfae1ab197780fb7a2dac92ad8a3d8f7ca8049a9cbe56a4218cd52e8da570"}, + {file = "mkdocs_autorefs-1.0.1.tar.gz", hash = "sha256:f684edf847eced40b570b57846b15f0bf57fb93ac2c510450775dcf16accb971"}, +] + +[package.dependencies] +Markdown = ">=3.3" +markupsafe = ">=2.0.1" +mkdocs = ">=1.1" + +[[package]] +name = "mkdocs-extra-sass-plugin" +version = "0.1.0" +description = "This plugin adds stylesheets to your mkdocs site from `Sass`/`SCSS`." +optional = false +python-versions = ">=3.6" +files = [ + {file = "mkdocs-extra-sass-plugin-0.1.0.tar.gz", hash = "sha256:cca7ae778585514371b22a63bcd69373d77e474edab4b270cf2924e05c879219"}, + {file = "mkdocs_extra_sass_plugin-0.1.0-py3-none-any.whl", hash = "sha256:10aa086fa8ef1fc4650f7bb6927deb7bf5bbf5a2dd3178f47e4ef44546b156db"}, +] + +[package.dependencies] +beautifulsoup4 = ">=4.6.3" +libsass = ">=0.15" +mkdocs = ">=1.1" + +[[package]] +name = "mkdocs-material" +version = "9.5.18" +description = "Documentation that simply works" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocs_material-9.5.18-py3-none-any.whl", hash = "sha256:1e0e27fc9fe239f9064318acf548771a4629d5fd5dfd45444fd80a953fe21eb4"}, + {file = "mkdocs_material-9.5.18.tar.gz", hash = "sha256:a43f470947053fa2405c33995f282d24992c752a50114f23f30da9d8d0c57e62"}, +] + +[package.dependencies] +babel = ">=2.10,<3.0" +colorama = ">=0.4,<1.0" +jinja2 = ">=3.0,<4.0" +markdown = ">=3.2,<4.0" +mkdocs = ">=1.5.3,<1.6.0" +mkdocs-material-extensions = ">=1.3,<2.0" +paginate = ">=0.5,<1.0" +pygments = ">=2.16,<3.0" +pymdown-extensions = ">=10.2,<11.0" +regex = ">=2022.4" +requests = ">=2.26,<3.0" + +[package.extras] +git = ["mkdocs-git-committers-plugin-2 (>=1.1,<2.0)", "mkdocs-git-revision-date-localized-plugin (>=1.2.4,<2.0)"] +imaging = ["cairosvg (>=2.6,<3.0)", "pillow (>=10.2,<11.0)"] +recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2.0)", "mkdocs-rss-plugin (>=1.6,<2.0)"] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +description = "Extension pack for Python Markdown and MkDocs Material." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, + {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, +] + +[[package]] +name = "mkdocstrings" +version = "0.24.3" +description = "Automatic documentation from sources, for MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings-0.24.3-py3-none-any.whl", hash = "sha256:5c9cf2a32958cd161d5428699b79c8b0988856b0d4a8c5baf8395fc1bf4087c3"}, + {file = "mkdocstrings-0.24.3.tar.gz", hash = "sha256:f327b234eb8d2551a306735436e157d0a22d45f79963c60a8b585d5f7a94c1d2"}, +] + +[package.dependencies] +click = ">=7.0" +importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} +Jinja2 = ">=2.11.1" +Markdown = ">=3.3" +MarkupSafe = ">=1.1" +mkdocs = ">=1.4" +mkdocs-autorefs = ">=0.3.1" +platformdirs = ">=2.2.0" +pymdown-extensions = ">=6.3" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.10\""} + +[package.extras] +crystal = ["mkdocstrings-crystal (>=0.3.4)"] +python = ["mkdocstrings-python (>=0.5.2)"] +python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] + +[[package]] +name = "mkdocstrings-python" +version = "1.10.0" +description = "A Python handler for mkdocstrings." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings_python-1.10.0-py3-none-any.whl", hash = "sha256:ba833fbd9d178a4b9d5cb2553a4df06e51dc1f51e41559a4d2398c16a6f69ecc"}, + {file = "mkdocstrings_python-1.10.0.tar.gz", hash = "sha256:71678fac657d4d2bb301eed4e4d2d91499c095fd1f8a90fa76422a87a5693828"}, +] + +[package.dependencies] +griffe = ">=0.44" +mkdocstrings = ">=0.24.2" + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "nodeenv" +version = "1.8.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "omegaconf" +version = "2.3.0" +description = "A flexible configuration library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, + {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, +] + +[package.dependencies] +antlr4-python3-runtime = "==4.9.*" +PyYAML = ">=5.1.0" + +[[package]] +name = "ordered-set" +version = "4.1.0" +description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8"}, + {file = "ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562"}, +] + +[package.extras] +dev = ["black", "mypy", "pytest"] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "paginate" +version = "0.5.6" +description = "Divides large result sets into pages for easier browsing" +optional = false +python-versions = "*" +files = [ + {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "poetry-dynamic-versioning" +version = "1.2.0" +description = "Plugin for Poetry to enable dynamic versioning based on VCS tags" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "poetry_dynamic_versioning-1.2.0-py3-none-any.whl", hash = "sha256:8dbef9728d866eb3d1a4edbb29c7ac8abdf96e3ca659473e950e2c016f3785ec"}, + {file = "poetry_dynamic_versioning-1.2.0.tar.gz", hash = "sha256:1a7bbdba2530499e73dfc6ac0af19de29020ab4aaa3e507573877114e6b71ed6"}, +] + +[package.dependencies] +dunamai = ">=1.18.0,<2.0.0" +jinja2 = ">=2.11.1,<4" +tomlkit = ">=0.4" + +[package.extras] +plugin = ["poetry (>=1.2.0,<2.0.0)"] + +[[package]] +name = "pre-commit" +version = "3.7.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"}, + {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "psutil" +version = "5.9.8" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"}, + {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"}, + {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"}, + {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, + {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"}, + {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"}, + {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, + {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, + {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, + {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + +[[package]] +name = "pycodestyle" +version = "2.11.1" +description = "Python style guide checker" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, +] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pycryptodome" +version = "3.20.0" +description = "Cryptographic library for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, + {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +] + +[[package]] +name = "pydantic" +version = "2.7.0" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.7.0-py3-none-any.whl", hash = "sha256:9dee74a271705f14f9a1567671d144a851c675b072736f0a7b2608fd9e495352"}, + {file = "pydantic-2.7.0.tar.gz", hash = "sha256:b5ecdd42262ca2462e2624793551e80911a1e989f462910bb81aef974b4bb383"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.18.1" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.18.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ee9cf33e7fe14243f5ca6977658eb7d1042caaa66847daacbd2117adb258b226"}, + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b7bbb97d82659ac8b37450c60ff2e9f97e4eb0f8a8a3645a5568b9334b08b50"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df4249b579e75094f7e9bb4bd28231acf55e308bf686b952f43100a5a0be394c"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0491006a6ad20507aec2be72e7831a42efc93193d2402018007ff827dc62926"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ae80f72bb7a3e397ab37b53a2b49c62cc5496412e71bc4f1277620a7ce3f52b"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58aca931bef83217fca7a390e0486ae327c4af9c3e941adb75f8772f8eeb03a1"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1be91ad664fc9245404a789d60cba1e91c26b1454ba136d2a1bf0c2ac0c0505a"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:667880321e916a8920ef49f5d50e7983792cf59f3b6079f3c9dac2b88a311d17"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f7054fdc556f5421f01e39cbb767d5ec5c1139ea98c3e5b350e02e62201740c7"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:030e4f9516f9947f38179249778709a460a3adb516bf39b5eb9066fcfe43d0e6"}, + {file = "pydantic_core-2.18.1-cp310-none-win32.whl", hash = "sha256:2e91711e36e229978d92642bfc3546333a9127ecebb3f2761372e096395fc649"}, + {file = "pydantic_core-2.18.1-cp310-none-win_amd64.whl", hash = "sha256:9a29726f91c6cb390b3c2338f0df5cd3e216ad7a938762d11c994bb37552edb0"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9ece8a49696669d483d206b4474c367852c44815fca23ac4e48b72b339807f80"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a5d83efc109ceddb99abd2c1316298ced2adb4570410defe766851a804fcd5b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7973c381283783cd1043a8c8f61ea5ce7a3a58b0369f0ee0ee975eaf2f2a1b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54c7375c62190a7845091f521add19b0f026bcf6ae674bdb89f296972272e86d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd63cec4e26e790b70544ae5cc48d11b515b09e05fdd5eff12e3195f54b8a586"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:561cf62c8a3498406495cfc49eee086ed2bb186d08bcc65812b75fda42c38294"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68717c38a68e37af87c4da20e08f3e27d7e4212e99e96c3d875fbf3f4812abfc"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d5728e93d28a3c63ee513d9ffbac9c5989de8c76e049dbcb5bfe4b923a9739d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0f17814c505f07806e22b28856c59ac80cee7dd0fbb152aed273e116378f519"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d816f44a51ba5175394bc6c7879ca0bd2be560b2c9e9f3411ef3a4cbe644c2e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win32.whl", hash = "sha256:09f03dfc0ef8c22622eaa8608caa4a1e189cfb83ce847045eca34f690895eccb"}, + {file = "pydantic_core-2.18.1-cp311-none-win_amd64.whl", hash = "sha256:27f1009dc292f3b7ca77feb3571c537276b9aad5dd4efb471ac88a8bd09024e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win_arm64.whl", hash = "sha256:48dd883db92e92519201f2b01cafa881e5f7125666141a49ffba8b9facc072b0"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b6b0e4912030c6f28bcb72b9ebe4989d6dc2eebcd2a9cdc35fefc38052dd4fe8"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3202a429fe825b699c57892d4371c74cc3456d8d71b7f35d6028c96dfecad31"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3982b0a32d0a88b3907e4b0dc36809fda477f0757c59a505d4e9b455f384b8b"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25595ac311f20e5324d1941909b0d12933f1fd2171075fcff763e90f43e92a0d"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14fe73881cf8e4cbdaded8ca0aa671635b597e42447fec7060d0868b52d074e6"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca976884ce34070799e4dfc6fbd68cb1d181db1eefe4a3a94798ddfb34b8867f"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684d840d2c9ec5de9cb397fcb3f36d5ebb6fa0d94734f9886032dd796c1ead06"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:54764c083bbe0264f0f746cefcded6cb08fbbaaf1ad1d78fb8a4c30cff999a90"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:201713f2f462e5c015b343e86e68bd8a530a4f76609b33d8f0ec65d2b921712a"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd1a9edb9dd9d79fbeac1ea1f9a8dd527a6113b18d2e9bcc0d541d308dae639b"}, + {file = "pydantic_core-2.18.1-cp312-none-win32.whl", hash = "sha256:d5e6b7155b8197b329dc787356cfd2684c9d6a6b1a197f6bbf45f5555a98d411"}, + {file = "pydantic_core-2.18.1-cp312-none-win_amd64.whl", hash = "sha256:9376d83d686ec62e8b19c0ac3bf8d28d8a5981d0df290196fb6ef24d8a26f0d6"}, + {file = "pydantic_core-2.18.1-cp312-none-win_arm64.whl", hash = "sha256:c562b49c96906b4029b5685075fe1ebd3b5cc2601dfa0b9e16c2c09d6cbce048"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3e352f0191d99fe617371096845070dee295444979efb8f27ad941227de6ad09"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0295d52b012cbe0d3059b1dba99159c3be55e632aae1999ab74ae2bd86a33d7"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56823a92075780582d1ffd4489a2e61d56fd3ebb4b40b713d63f96dd92d28144"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd3f79e17b56741b5177bcc36307750d50ea0698df6aa82f69c7db32d968c1c2"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38a5024de321d672a132b1834a66eeb7931959c59964b777e8f32dbe9523f6b1"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ce426ee691319d4767748c8e0895cfc56593d725594e415f274059bcf3cb76"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2adaeea59849ec0939af5c5d476935f2bab4b7f0335b0110f0f069a41024278e"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9b6431559676a1079eac0f52d6d0721fb8e3c5ba43c37bc537c8c83724031feb"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:85233abb44bc18d16e72dc05bf13848a36f363f83757541f1a97db2f8d58cfd9"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:641a018af4fe48be57a2b3d7a1f0f5dbca07c1d00951d3d7463f0ac9dac66622"}, + {file = "pydantic_core-2.18.1-cp38-none-win32.whl", hash = "sha256:63d7523cd95d2fde0d28dc42968ac731b5bb1e516cc56b93a50ab293f4daeaad"}, + {file = "pydantic_core-2.18.1-cp38-none-win_amd64.whl", hash = "sha256:907a4d7720abfcb1c81619863efd47c8a85d26a257a2dbebdb87c3b847df0278"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:aad17e462f42ddbef5984d70c40bfc4146c322a2da79715932cd8976317054de"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:94b9769ba435b598b547c762184bcfc4783d0d4c7771b04a3b45775c3589ca44"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e0e57cc704a52fb1b48f16d5b2c8818da087dbee6f98d9bf19546930dc64b5"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76b86e24039c35280ceee6dce7e62945eb93a5175d43689ba98360ab31eebc4a"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12a05db5013ec0ca4a32cc6433f53faa2a014ec364031408540ba858c2172bb0"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:250ae39445cb5475e483a36b1061af1bc233de3e9ad0f4f76a71b66231b07f88"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a32204489259786a923e02990249c65b0f17235073149d0033efcebe80095570"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6395a4435fa26519fd96fdccb77e9d00ddae9dd6c742309bd0b5610609ad7fb2"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2533ad2883f001efa72f3d0e733fb846710c3af6dcdd544fe5bf14fa5fe2d7db"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b560b72ed4816aee52783c66854d96157fd8175631f01ef58e894cc57c84f0f6"}, + {file = "pydantic_core-2.18.1-cp39-none-win32.whl", hash = "sha256:582cf2cead97c9e382a7f4d3b744cf0ef1a6e815e44d3aa81af3ad98762f5a9b"}, + {file = "pydantic_core-2.18.1-cp39-none-win_amd64.whl", hash = "sha256:ca71d501629d1fa50ea7fa3b08ba884fe10cefc559f5c6c8dfe9036c16e8ae89"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e178e5b66a06ec5bf51668ec0d4ac8cfb2bdcb553b2c207d58148340efd00143"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:72722ce529a76a4637a60be18bd789d8fb871e84472490ed7ddff62d5fed620d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fe0c1ce5b129455e43f941f7a46f61f3d3861e571f2905d55cdbb8b5c6f5e2c"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4284c621f06a72ce2cb55f74ea3150113d926a6eb78ab38340c08f770eb9b4d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0c3e718f4e064efde68092d9d974e39572c14e56726ecfaeebbe6544521f47"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2027493cc44c23b598cfaf200936110433d9caa84e2c6cf487a83999638a96ac"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76909849d1a6bffa5a07742294f3fa1d357dc917cb1fe7b470afbc3a7579d539"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ee7ccc7fb7e921d767f853b47814c3048c7de536663e82fbc37f5eb0d532224b"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ee2794111c188548a4547eccc73a6a8527fe2af6cf25e1a4ebda2fd01cdd2e60"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a139fe9f298dc097349fb4f28c8b81cc7a202dbfba66af0e14be5cfca4ef7ce5"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d074b07a10c391fc5bbdcb37b2f16f20fcd9e51e10d01652ab298c0d07908ee2"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c69567ddbac186e8c0aadc1f324a60a564cfe25e43ef2ce81bcc4b8c3abffbae"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf1c7b78cddb5af00971ad5294a4583188bda1495b13760d9f03c9483bb6203"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2684a94fdfd1b146ff10689c6e4e815f6a01141781c493b97342cdc5b06f4d5d"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:73c1bc8a86a5c9e8721a088df234265317692d0b5cd9e86e975ce3bc3db62a59"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e60defc3c15defb70bb38dd605ff7e0fae5f6c9c7cbfe0ad7868582cb7e844a6"}, + {file = "pydantic_core-2.18.1.tar.gz", hash = "sha256:de9d3e8717560eb05e28739d1b35e4eac2e458553a52a301e51352a7ffc86a35"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyflakes" +version = "3.2.0" +description = "passive checker of Python programs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, +] + +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pymdown-extensions" +version = "10.8" +description = "Extension pack for Python Markdown." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pymdown_extensions-10.8-py3-none-any.whl", hash = "sha256:3539003ff0d5e219ba979d2dc961d18fcad5ac259e66c764482e8347b4c0503c"}, + {file = "pymdown_extensions-10.8.tar.gz", hash = "sha256:91ca336caf414e1e5e0626feca86e145de9f85a3921a7bcbd32890b51738c428"}, +] + +[package.dependencies] +markdown = ">=3.6" +pyyaml = "*" + +[package.extras] +extra = ["pygments (>=2.12)"] + +[[package]] +name = "pytest" +version = "8.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, + {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.4,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.23.6" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-asyncio-0.23.6.tar.gz", hash = "sha256:ffe523a89c1c222598c76856e76852b787504ddb72dd5d9b6617ffa8aa2cde5f"}, + {file = "pytest_asyncio-0.23.6-py3-none-any.whl", hash = "sha256:68516fdd1018ac57b846c9846b954f0393b26f094764a28c955eabb0536a4e8a"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + +[[package]] +name = "pytest-cov" +version = "5.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, + {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "pytest-env" +version = "1.1.3" +description = "pytest plugin that allows you to add environment variables." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_env-1.1.3-py3-none-any.whl", hash = "sha256:aada77e6d09fcfb04540a6e462c58533c37df35fa853da78707b17ec04d17dfc"}, + {file = "pytest_env-1.1.3.tar.gz", hash = "sha256:fcd7dc23bb71efd3d35632bde1bbe5ee8c8dc4489d6617fb010674880d96216b"}, +] + +[package.dependencies] +pytest = ">=7.4.3" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +test = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pytest-httpserver" +version = "1.0.10" +description = "pytest-httpserver is a httpserver for pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_httpserver-1.0.10-py3-none-any.whl", hash = "sha256:d40e0cc3d61ed6e4d80f52a796926d557a7db62b17e43b3e258a78a3c34becb9"}, + {file = "pytest_httpserver-1.0.10.tar.gz", hash = "sha256:77b9fbc2eb0a129cfbbacc8fe57e8cafe071d506489f31fe31e62f1b332d9905"}, +] + +[package.dependencies] +Werkzeug = ">=2.0.0" + +[[package]] +name = "pytest-httpx" +version = "0.29.0" +description = "Send responses to httpx." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest_httpx-0.29.0-py3-none-any.whl", hash = "sha256:7d6fd29042e7b98ed98199ded120bc8100c8078ca306952666e89bf8807b95ff"}, + {file = "pytest_httpx-0.29.0.tar.gz", hash = "sha256:ed08ed802e2b315b83cdd16f0b26cbb2b836c29e0fde5c18bc3105f1073e0332"}, +] + +[package.dependencies] +httpx = "==0.26.*" +pytest = ">=7,<9" + +[package.extras] +testing = ["pytest-asyncio (==0.23.*)", "pytest-cov (==4.*)"] + +[[package]] +name = "pytest-rerunfailures" +version = "14.0" +description = "pytest plugin to re-run tests to eliminate flaky failures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-rerunfailures-14.0.tar.gz", hash = "sha256:4a400bcbcd3c7a4ad151ab8afac123d90eca3abe27f98725dc4d9702887d2e92"}, + {file = "pytest_rerunfailures-14.0-py3-none-any.whl", hash = "sha256:4197bdd2eaeffdbf50b5ea6e7236f47ff0e44d1def8dae08e409f536d84e7b32"}, +] + +[package.dependencies] +packaging = ">=17.1" +pytest = ">=7.2" + +[[package]] +name = "pytest-timeout" +version = "2.3.1" +description = "pytest plugin to abort hanging tests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-timeout-2.3.1.tar.gz", hash = "sha256:12397729125c6ecbdaca01035b9e5239d4db97352320af155b3f5de1ba5165d9"}, + {file = "pytest_timeout-2.3.1-py3-none-any.whl", hash = "sha256:68188cb703edfc6a18fad98dc25a3c61e9f24d644b0b70f33af545219fc7813e"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[[package]] +name = "python-daemon" +version = "3.0.1" +description = "Library to implement a well-behaved Unix daemon process." +optional = false +python-versions = ">=3" +files = [ + {file = "python-daemon-3.0.1.tar.gz", hash = "sha256:6c57452372f7eaff40934a1c03ad1826bf5e793558e87fef49131e6464b4dae5"}, + {file = "python_daemon-3.0.1-py3-none-any.whl", hash = "sha256:42bb848a3260a027fa71ad47ecd959e471327cb34da5965962edd5926229f341"}, +] + +[package.dependencies] +docutils = "*" +lockfile = ">=0.10" +setuptools = ">=62.4.0" + +[package.extras] +devel = ["coverage", "docutils", "isort", "testscenarios (>=0.4)", "testtools", "twine"] +test = ["coverage", "docutils", "testscenarios (>=0.4)", "testtools"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "pyyaml-env-tag" +version = "0.1" +description = "A custom YAML tag for referencing environment variables in YAML files. " +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, + {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, +] + +[package.dependencies] +pyyaml = "*" + +[[package]] +name = "regex" +version = "2024.4.16" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.7" +files = [ + {file = "regex-2024.4.16-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb83cc090eac63c006871fd24db5e30a1f282faa46328572661c0a24a2323a08"}, + {file = "regex-2024.4.16-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c91e1763696c0eb66340c4df98623c2d4e77d0746b8f8f2bee2c6883fd1fe18"}, + {file = "regex-2024.4.16-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:10188fe732dec829c7acca7422cdd1bf57d853c7199d5a9e96bb4d40db239c73"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:956b58d692f235cfbf5b4f3abd6d99bf102f161ccfe20d2fd0904f51c72c4c66"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a70b51f55fd954d1f194271695821dd62054d949efd6368d8be64edd37f55c86"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c02fcd2bf45162280613d2e4a1ca3ac558ff921ae4e308ecb307650d3a6ee51"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ed75ea6892a56896d78f11006161eea52c45a14994794bcfa1654430984b22"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd727ad276bb91928879f3aa6396c9a1d34e5e180dce40578421a691eeb77f47"}, + {file = "regex-2024.4.16-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7cbc5d9e8a1781e7be17da67b92580d6ce4dcef5819c1b1b89f49d9678cc278c"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:78fddb22b9ef810b63ef341c9fcf6455232d97cfe03938cbc29e2672c436670e"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:445ca8d3c5a01309633a0c9db57150312a181146315693273e35d936472df912"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:95399831a206211d6bc40224af1c635cb8790ddd5c7493e0bd03b85711076a53"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:7731728b6568fc286d86745f27f07266de49603a6fdc4d19c87e8c247be452af"}, + {file = "regex-2024.4.16-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4facc913e10bdba42ec0aee76d029aedda628161a7ce4116b16680a0413f658a"}, + {file = "regex-2024.4.16-cp310-cp310-win32.whl", hash = "sha256:911742856ce98d879acbea33fcc03c1d8dc1106234c5e7d068932c945db209c0"}, + {file = "regex-2024.4.16-cp310-cp310-win_amd64.whl", hash = "sha256:e0a2df336d1135a0b3a67f3bbf78a75f69562c1199ed9935372b82215cddd6e2"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1210365faba7c2150451eb78ec5687871c796b0f1fa701bfd2a4a25420482d26"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9ab40412f8cd6f615bfedea40c8bf0407d41bf83b96f6fc9ff34976d6b7037fd"}, + {file = "regex-2024.4.16-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fd80d1280d473500d8086d104962a82d77bfbf2b118053824b7be28cd5a79ea5"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb966fdd9217e53abf824f437a5a2d643a38d4fd5fd0ca711b9da683d452969"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20b7a68444f536365af42a75ccecb7ab41a896a04acf58432db9e206f4e525d6"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b74586dd0b039c62416034f811d7ee62810174bb70dffcca6439f5236249eb09"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c8290b44d8b0af4e77048646c10c6e3aa583c1ca67f3b5ffb6e06cf0c6f0f89"}, + {file = "regex-2024.4.16-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2d80a6749724b37853ece57988b39c4e79d2b5fe2869a86e8aeae3bbeef9eb0"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3a1018e97aeb24e4f939afcd88211ace472ba566efc5bdf53fd8fd7f41fa7170"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8d015604ee6204e76569d2f44e5a210728fa917115bef0d102f4107e622b08d5"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:3d5ac5234fb5053850d79dd8eb1015cb0d7d9ed951fa37aa9e6249a19aa4f336"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:0a38d151e2cdd66d16dab550c22f9521ba79761423b87c01dae0a6e9add79c0d"}, + {file = "regex-2024.4.16-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:159dc4e59a159cb8e4e8f8961eb1fa5d58f93cb1acd1701d8aff38d45e1a84a6"}, + {file = "regex-2024.4.16-cp311-cp311-win32.whl", hash = "sha256:ba2336d6548dee3117520545cfe44dc28a250aa091f8281d28804aa8d707d93d"}, + {file = "regex-2024.4.16-cp311-cp311-win_amd64.whl", hash = "sha256:8f83b6fd3dc3ba94d2b22717f9c8b8512354fd95221ac661784df2769ea9bba9"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:80b696e8972b81edf0af2a259e1b2a4a661f818fae22e5fa4fa1a995fb4a40fd"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d61ae114d2a2311f61d90c2ef1358518e8f05eafda76eaf9c772a077e0b465ec"}, + {file = "regex-2024.4.16-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ba6745440b9a27336443b0c285d705ce73adb9ec90e2f2004c64d95ab5a7598"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295004b2dd37b0835ea5c14a33e00e8cfa3c4add4d587b77287825f3418d310"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aba818dcc7263852aabb172ec27b71d2abca02a593b95fa79351b2774eb1d2b"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0800631e565c47520aaa04ae38b96abc5196fe8b4aa9bd864445bd2b5848a7a"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08dea89f859c3df48a440dbdcd7b7155bc675f2fa2ec8c521d02dc69e877db70"}, + {file = "regex-2024.4.16-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eeaa0b5328b785abc344acc6241cffde50dc394a0644a968add75fcefe15b9d4"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4e819a806420bc010489f4e741b3036071aba209f2e0989d4750b08b12a9343f"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c2d0e7cbb6341e830adcbfa2479fdeebbfbb328f11edd6b5675674e7a1e37730"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:91797b98f5e34b6a49f54be33f72e2fb658018ae532be2f79f7c63b4ae225145"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:d2da13568eff02b30fd54fccd1e042a70fe920d816616fda4bf54ec705668d81"}, + {file = "regex-2024.4.16-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:370c68dc5570b394cbaadff50e64d705f64debed30573e5c313c360689b6aadc"}, + {file = "regex-2024.4.16-cp312-cp312-win32.whl", hash = "sha256:904c883cf10a975b02ab3478bce652f0f5346a2c28d0a8521d97bb23c323cc8b"}, + {file = "regex-2024.4.16-cp312-cp312-win_amd64.whl", hash = "sha256:785c071c982dce54d44ea0b79cd6dfafddeccdd98cfa5f7b86ef69b381b457d9"}, + {file = "regex-2024.4.16-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2f142b45c6fed48166faeb4303b4b58c9fcd827da63f4cf0a123c3480ae11fb"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e87ab229332ceb127a165612d839ab87795972102cb9830e5f12b8c9a5c1b508"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81500ed5af2090b4a9157a59dbc89873a25c33db1bb9a8cf123837dcc9765047"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b340cccad138ecb363324aa26893963dcabb02bb25e440ebdf42e30963f1a4e0"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c72608e70f053643437bd2be0608f7f1c46d4022e4104d76826f0839199347a"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a01fe2305e6232ef3e8f40bfc0f0f3a04def9aab514910fa4203bafbc0bb4682"}, + {file = "regex-2024.4.16-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:03576e3a423d19dda13e55598f0fd507b5d660d42c51b02df4e0d97824fdcae3"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:549c3584993772e25f02d0656ac48abdda73169fe347263948cf2b1cead622f3"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:34422d5a69a60b7e9a07a690094e824b66f5ddc662a5fc600d65b7c174a05f04"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:5f580c651a72b75c39e311343fe6875d6f58cf51c471a97f15a938d9fe4e0d37"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3399dd8a7495bbb2bacd59b84840eef9057826c664472e86c91d675d007137f5"}, + {file = "regex-2024.4.16-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d1f86f3f4e2388aa3310b50694ac44daefbd1681def26b4519bd050a398dc5a"}, + {file = "regex-2024.4.16-cp37-cp37m-win32.whl", hash = "sha256:dd5acc0a7d38fdc7a3a6fd3ad14c880819008ecb3379626e56b163165162cc46"}, + {file = "regex-2024.4.16-cp37-cp37m-win_amd64.whl", hash = "sha256:ba8122e3bb94ecda29a8de4cf889f600171424ea586847aa92c334772d200331"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:743deffdf3b3481da32e8a96887e2aa945ec6685af1cfe2bcc292638c9ba2f48"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7571f19f4a3fd00af9341c7801d1ad1967fc9c3f5e62402683047e7166b9f2b4"}, + {file = "regex-2024.4.16-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:df79012ebf6f4efb8d307b1328226aef24ca446b3ff8d0e30202d7ebcb977a8c"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e757d475953269fbf4b441207bb7dbdd1c43180711b6208e129b637792ac0b93"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4313ab9bf6a81206c8ac28fdfcddc0435299dc88cad12cc6305fd0e78b81f9e4"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d83c2bc678453646f1a18f8db1e927a2d3f4935031b9ad8a76e56760461105dd"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9df1bfef97db938469ef0a7354b2d591a2d438bc497b2c489471bec0e6baf7c4"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62120ed0de69b3649cc68e2965376048793f466c5a6c4370fb27c16c1beac22d"}, + {file = "regex-2024.4.16-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c2ef6f7990b6e8758fe48ad08f7e2f66c8f11dc66e24093304b87cae9037bb4a"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8fc6976a3395fe4d1fbeb984adaa8ec652a1e12f36b56ec8c236e5117b585427"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:03e68f44340528111067cecf12721c3df4811c67268b897fbe695c95f860ac42"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ec7e0043b91115f427998febaa2beb82c82df708168b35ece3accb610b91fac1"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c21fc21a4c7480479d12fd8e679b699f744f76bb05f53a1d14182b31f55aac76"}, + {file = "regex-2024.4.16-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:12f6a3f2f58bb7344751919a1876ee1b976fe08b9ffccb4bbea66f26af6017b9"}, + {file = "regex-2024.4.16-cp38-cp38-win32.whl", hash = "sha256:479595a4fbe9ed8f8f72c59717e8cf222da2e4c07b6ae5b65411e6302af9708e"}, + {file = "regex-2024.4.16-cp38-cp38-win_amd64.whl", hash = "sha256:0534b034fba6101611968fae8e856c1698da97ce2efb5c2b895fc8b9e23a5834"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a7ccdd1c4a3472a7533b0a7aa9ee34c9a2bef859ba86deec07aff2ad7e0c3b94"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6f2f017c5be19984fbbf55f8af6caba25e62c71293213f044da3ada7091a4455"}, + {file = "regex-2024.4.16-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:803b8905b52de78b173d3c1e83df0efb929621e7b7c5766c0843704d5332682f"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:684008ec44ad275832a5a152f6e764bbe1914bea10968017b6feaecdad5736e0"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65436dce9fdc0aeeb0a0effe0839cb3d6a05f45aa45a4d9f9c60989beca78b9c"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea355eb43b11764cf799dda62c658c4d2fdb16af41f59bb1ccfec517b60bcb07"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c1165f3809ce7774f05cb74e5408cd3aa93ee8573ae959a97a53db3ca3180d"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cccc79a9be9b64c881f18305a7c715ba199e471a3973faeb7ba84172abb3f317"}, + {file = "regex-2024.4.16-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00169caa125f35d1bca6045d65a662af0202704489fada95346cfa092ec23f39"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6cc38067209354e16c5609b66285af17a2863a47585bcf75285cab33d4c3b8df"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:23cff1b267038501b179ccbbd74a821ac4a7192a1852d1d558e562b507d46013"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d320b3bf82a39f248769fc7f188e00f93526cc0fe739cfa197868633d44701"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:89ec7f2c08937421bbbb8b48c54096fa4f88347946d4747021ad85f1b3021b3c"}, + {file = "regex-2024.4.16-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4918fd5f8b43aa7ec031e0fef1ee02deb80b6afd49c85f0790be1dc4ce34cb50"}, + {file = "regex-2024.4.16-cp39-cp39-win32.whl", hash = "sha256:684e52023aec43bdf0250e843e1fdd6febbe831bd9d52da72333fa201aaa2335"}, + {file = "regex-2024.4.16-cp39-cp39-win_amd64.whl", hash = "sha256:e697e1c0238133589e00c244a8b676bc2cfc3ab4961318d902040d099fec7483"}, + {file = "regex-2024.4.16.tar.gz", hash = "sha256:fa454d26f2e87ad661c4f0c5a5fe4cf6aab1e307d1b94f16ffdfcb089ba685c0"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-file" +version = "2.0.0" +description = "File transport adapter for Requests" +optional = false +python-versions = "*" +files = [ + {file = "requests-file-2.0.0.tar.gz", hash = "sha256:20c5931629c558fda566cacc10cfe2cd502433e628f568c34c80d96a0cc95972"}, +] + +[package.dependencies] +requests = ">=1.0.0" + +[[package]] +name = "resolvelib" +version = "1.0.1" +description = "Resolve abstract dependencies into concrete ones" +optional = false +python-versions = "*" +files = [ + {file = "resolvelib-1.0.1-py2.py3-none-any.whl", hash = "sha256:d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf"}, + {file = "resolvelib-1.0.1.tar.gz", hash = "sha256:04ce76cbd63fded2078ce224785da6ecd42b9564b1390793f64ddecbe997b309"}, +] + +[package.extras] +examples = ["html5lib", "packaging", "pygraphviz", "requests"] +lint = ["black", "flake8", "isort", "mypy", "types-requests"] +release = ["build", "towncrier", "twine"] +test = ["commentjson", "packaging", "pytest"] + +[[package]] +name = "setuptools" +version = "69.5.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "socksio" +version = "1.0.0" +description = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." +optional = false +python-versions = ">=3.6" +files = [ + {file = "socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3"}, + {file = "socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac"}, +] + +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + +[[package]] +name = "tabulate" +version = "0.8.10" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"}, + {file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "tldextract" +version = "5.1.2" +description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tldextract-5.1.2-py3-none-any.whl", hash = "sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46"}, + {file = "tldextract-5.1.2.tar.gz", hash = "sha256:c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200"}, +] + +[package.dependencies] +filelock = ">=3.0.8" +idna = "*" +requests = ">=2.1.0" +requests-file = ">=1.4" + +[package.extras] +release = ["build", "twine"] +testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "types-filelock", "types-requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.12.4" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.4-py3-none-any.whl", hash = "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b"}, + {file = "tomlkit-0.12.4.tar.gz", hash = "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3"}, +] + +[[package]] +name = "tornado" +version = "6.4" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, + {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, + {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, + {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, + {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, +] + +[[package]] +name = "typing-extensions" +version = "4.11.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, +] + +[[package]] +name = "unidecode" +version = "1.3.8" +description = "ASCII transliterations of Unicode text" +optional = false +python-versions = ">=3.5" +files = [ + {file = "Unidecode-1.3.8-py3-none-any.whl", hash = "sha256:d130a61ce6696f8148a3bd8fe779c99adeb4b870584eeb9526584e9aa091fd39"}, + {file = "Unidecode-1.3.8.tar.gz", hash = "sha256:cfdb349d46ed3873ece4586b96aa75258726e2fa8ec21d6f00a591d98806c2f4"}, +] + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "virtualenv" +version = "20.25.3" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.25.3-py3-none-any.whl", hash = "sha256:8aac4332f2ea6ef519c648d0bc48a5b1d324994753519919bddbb1aff25a104e"}, + {file = "virtualenv-20.25.3.tar.gz", hash = "sha256:7bb554bbdfeaacc3349fa614ea5bff6ac300fc7c335e9facf3a3bcfc703f45be"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + +[[package]] +name = "watchdog" +version = "4.0.0" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchdog-4.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:39cb34b1f1afbf23e9562501673e7146777efe95da24fab5707b88f7fb11649b"}, + {file = "watchdog-4.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c522392acc5e962bcac3b22b9592493ffd06d1fc5d755954e6be9f4990de932b"}, + {file = "watchdog-4.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6c47bdd680009b11c9ac382163e05ca43baf4127954c5f6d0250e7d772d2b80c"}, + {file = "watchdog-4.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8350d4055505412a426b6ad8c521bc7d367d1637a762c70fdd93a3a0d595990b"}, + {file = "watchdog-4.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c17d98799f32e3f55f181f19dd2021d762eb38fdd381b4a748b9f5a36738e935"}, + {file = "watchdog-4.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4986db5e8880b0e6b7cd52ba36255d4793bf5cdc95bd6264806c233173b1ec0b"}, + {file = "watchdog-4.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:11e12fafb13372e18ca1bbf12d50f593e7280646687463dd47730fd4f4d5d257"}, + {file = "watchdog-4.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5369136a6474678e02426bd984466343924d1df8e2fd94a9b443cb7e3aa20d19"}, + {file = "watchdog-4.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76ad8484379695f3fe46228962017a7e1337e9acadafed67eb20aabb175df98b"}, + {file = "watchdog-4.0.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:45cc09cc4c3b43fb10b59ef4d07318d9a3ecdbff03abd2e36e77b6dd9f9a5c85"}, + {file = "watchdog-4.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eed82cdf79cd7f0232e2fdc1ad05b06a5e102a43e331f7d041e5f0e0a34a51c4"}, + {file = "watchdog-4.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ba30a896166f0fee83183cec913298151b73164160d965af2e93a20bbd2ab605"}, + {file = "watchdog-4.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d18d7f18a47de6863cd480734613502904611730f8def45fc52a5d97503e5101"}, + {file = "watchdog-4.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2895bf0518361a9728773083908801a376743bcc37dfa252b801af8fd281b1ca"}, + {file = "watchdog-4.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87e9df830022488e235dd601478c15ad73a0389628588ba0b028cb74eb72fed8"}, + {file = "watchdog-4.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6e949a8a94186bced05b6508faa61b7adacc911115664ccb1923b9ad1f1ccf7b"}, + {file = "watchdog-4.0.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6a4db54edea37d1058b08947c789a2354ee02972ed5d1e0dca9b0b820f4c7f92"}, + {file = "watchdog-4.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d31481ccf4694a8416b681544c23bd271f5a123162ab603c7d7d2dd7dd901a07"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8fec441f5adcf81dd240a5fe78e3d83767999771630b5ddfc5867827a34fa3d3"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:6a9c71a0b02985b4b0b6d14b875a6c86ddea2fdbebd0c9a720a806a8bbffc69f"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:557ba04c816d23ce98a06e70af6abaa0485f6d94994ec78a42b05d1c03dcbd50"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:d0f9bd1fd919134d459d8abf954f63886745f4660ef66480b9d753a7c9d40927"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:f9b2fdca47dc855516b2d66eef3c39f2672cbf7e7a42e7e67ad2cbfcd6ba107d"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:73c7a935e62033bd5e8f0da33a4dcb763da2361921a69a5a95aaf6c93aa03a87"}, + {file = "watchdog-4.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6a80d5cae8c265842c7419c560b9961561556c4361b297b4c431903f8c33b269"}, + {file = "watchdog-4.0.0-py3-none-win32.whl", hash = "sha256:8f9a542c979df62098ae9c58b19e03ad3df1c9d8c6895d96c0d51da17b243b1c"}, + {file = "watchdog-4.0.0-py3-none-win_amd64.whl", hash = "sha256:f970663fa4f7e80401a7b0cbeec00fa801bf0287d93d48368fc3e6fa32716245"}, + {file = "watchdog-4.0.0-py3-none-win_ia64.whl", hash = "sha256:9a03e16e55465177d416699331b0f3564138f1807ecc5f2de9d55d8f188d08c7"}, + {file = "watchdog-4.0.0.tar.gz", hash = "sha256:e3e7065cbdabe6183ab82199d7a4f6b3ba0a438c5a512a68559846ccb76a78ec"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + +[[package]] +name = "websockets" +version = "12.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, +] + +[[package]] +name = "werkzeug" +version = "3.0.2" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-3.0.2-py3-none-any.whl", hash = "sha256:3aac3f5da756f93030740bc235d3e09449efcf65f2f55e3602e1d851b8f48795"}, + {file = "werkzeug-3.0.2.tar.gz", hash = "sha256:e39b645a6ac92822588e7b39a692e7828724ceae0b0d702ef96701f90e70128d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + +[[package]] +name = "wordninja" +version = "2.0.0" +description = "Probabilistically split concatenated words using NLP based on English Wikipedia uni-gram frequencies." +optional = false +python-versions = "*" +files = [ + {file = "wordninja-2.0.0.tar.gz", hash = "sha256:1a1cc7ec146ad19d6f71941ee82aef3d31221700f0d8bf844136cf8df79d281a"}, +] + +[[package]] +name = "xmltodict" +version = "0.12.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "xmltodict-0.12.0-py2.py3-none-any.whl", hash = "sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051"}, + {file = "xmltodict-0.12.0.tar.gz", hash = "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21"}, +] + +[[package]] +name = "xmltojson" +version = "2.0.2" +description = "A Python module and cli tool to quickly convert xml text or files into json" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "xmltojson-2.0.2-py3-none-any.whl", hash = "sha256:8ba5c8b33a5a0f824ad754ed62367d841ce91f7deaf82e118c28e42a0e24454c"}, + {file = "xmltojson-2.0.2.tar.gz", hash = "sha256:10719660409bd1825507e04d2fa4848c10591a092613bcd66651c7e0774f5405"}, +] + +[package.dependencies] +xmltodict = ">=0.12.0,<0.13.0" + +[[package]] +name = "zipp" +version = "3.18.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, + {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "1c2094963a62ff66cdbf77783459a5d2b163faf1b7b5da1f92fac6852e757a07" From 3b460fda7b72f3e9d3efe582ee654c26a7949538 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 11:45:04 -0400 Subject: [PATCH 154/171] fix merge --- bbot/scanner/target.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index e5577de91..a0f8130c8 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -81,16 +81,11 @@ def __init__(self, *targets, strict_scope=False): - Each target is processed and stored as an `Event` in the '_events' dictionary. """ self.strict_scope = strict_scope -<<<<<<< HEAD -======= - self.make_in_scope = make_in_scope self.special_event_types = { "ORG_STUB": re.compile(r"^ORG:(.*)", re.IGNORECASE), "ASN": re.compile(r"^ASN:(.*)", re.IGNORECASE), } - self._dummy_module = TargetDummyModule(scan) ->>>>>>> dev self._events = dict() if len(targets) > 0: log.verbose(f"Creating events from {len(targets):,} targets") From 0f6240c3e2e2df625cc64621c986b48d9a1f29e0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 13:47:07 -0400 Subject: [PATCH 155/171] update cloudcheck --- bbot/modules/internal/cloud.py | 3 ++- bbot/test/test_step_2/module_tests/test_module_cloud.py | 2 ++ poetry.lock | 9 +++++---- pyproject.toml | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index 14aaba930..e6bab4baa 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -9,8 +9,9 @@ class cloud(InterceptModule): async def setup(self): self.dummy_modules = {} - for provider_name in self.helpers.cloud.providers: + for provider_name, provider in self.helpers.cloud.providers.items(): self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan") + return True async def filter_event(self, event): diff --git a/bbot/test/test_step_2/module_tests/test_module_cloud.py b/bbot/test/test_step_2/module_tests/test_module_cloud.py index 1d4e59283..1ee8df5e7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_cloud.py +++ b/bbot/test/test_step_2/module_tests/test_module_cloud.py @@ -70,6 +70,8 @@ async def setup_after_prep(self, module_test): assert "cloud-storage-bucket" in google_event3.tags def check(self, module_test, events): + for e in events: + self.log.debug(e) assert 2 == len([e for e in events if e.type == "STORAGE_BUCKET"]) assert 1 == len( [ diff --git a/poetry.lock b/poetry.lock index ce4a80b8f..05386f0a4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -388,18 +388,19 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "3.1.0.318" +version = "4.0.0.345" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-3.1.0.318-py3-none-any.whl", hash = "sha256:471dba97531e1f60aadab8daa6cb1d63727f67c16fd7b4758db46c9af2f362f1"}, - {file = "cloudcheck-3.1.0.318.tar.gz", hash = "sha256:ba7fcc026817aa05f74c7789d2ac306469f3143f91b3ea9f95c57c70a7b0b787"}, + {file = "cloudcheck-4.0.0.345-py3-none-any.whl", hash = "sha256:82a1cecaa0ec35a50d6c1e4884a9535eb4c1c788b845b0c4a91b44935f4dc765"}, + {file = "cloudcheck-4.0.0.345.tar.gz", hash = "sha256:787953a305c0be6e6eb4ceb9990dccb633f9e1429d5ebfda7acf7dca35b3caeb"}, ] [package.dependencies] httpx = ">=0.26,<0.28" pydantic = ">=2.4.2,<3.0.0" +regex = ">=2024.4.16,<2025.0.0" [[package]] name = "colorama" @@ -2624,4 +2625,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "6e455b3aa900eff3b432ee7f9f92ca1a13193eccbd7c75bee99cb4d12891dfdd" +content-hash = "15633b02fcedb3d044f4e40a45ce1e9dd7209608a0389175a4523e3810a8504b" diff --git a/pyproject.toml b/pyproject.toml index 90bf19d47..0cc6eed31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ lxml = ">=4.9.2,<6.0.0" dnspython = "^2.4.2" pydantic = "^2.4.2" httpx = "^0.26.0" -cloudcheck = ">=2.1.0.181,<4.0.0.0" tldextract = "^5.1.1" cachetools = "^5.3.2" socksio = "^1.0.0" @@ -51,6 +50,7 @@ jinja2 = "^3.1.3" pyzmq = "^25.1.2" regex = "^2024.4.16" unidecode = "^1.3.8" +cloudcheck = "^4.0.0.345" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From e6c507f2ecaef0e7d216b80d154a5527ad509557 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 13:52:50 -0400 Subject: [PATCH 156/171] tests for custom target types --- bbot/test/test_step_1/test_cli.py | 8 ++++++++ bbot/test/test_step_1/test_python_api.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 6ada6e64e..4a42b451a 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -190,6 +190,14 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): assert "| active" in caplog.text assert not "| passive" in caplog.text + # custom target type + caplog.clear() + assert not caplog.text + monkeypatch.setattr("sys.argv", ["bbot", "-t", "ORG:evilcorp"]) + result = await cli._main() + assert result == True + assert "[ORG_STUB] evilcorp TARGET" in caplog.text + # no args caplog.clear() assert not caplog.text diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 14c2fae1d..0155dcfb3 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -45,6 +45,11 @@ async def test_python_api(): Scanner("127.0.0.1", config={"home": bbot_home}) assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools") + # custom target types + custom_target_scan = Scanner("ORG:evilcorp") + events = [e async for e in custom_target_scan.async_start()] + assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp" and "target" in e.tags]) + def test_python_api_sync(): from bbot.scanner import Scanner From a6a7ade89b4ead0f38c3af6a10fd360ed981120c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Apr 2024 15:15:09 -0400 Subject: [PATCH 157/171] fix tests --- bbot/modules/hunt.py | 2 +- bbot/modules/paramminer_headers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index dd591a345..0a759f2cf 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -280,7 +280,7 @@ class hunt(BaseModule): async def handle_event(self, event): body = event.data.get("body", "") - for p in await self.helpers.extract_params_html(body): + for p in await self.helpers.re.extract_params_html(body): for k in hunt_param_dict.keys(): if p.lower() in hunt_param_dict[k]: description = f"Found potential {k.upper()} parameter [{p}]" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 35b5c0df1..561a05fe2 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -203,7 +203,7 @@ async def load_extracted_words(self, body, content_type): elif content_type and "xml" in content_type.lower(): return extract_params_xml(body) else: - return set(await self.helpers.extract_params_html(body)) + return set(await self.helpers.re.extract_params_html(body)) async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False): if reasons is None: From eaf2cdf6f206521470682e0fc6c8087b067c5c72 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:23:18 -0400 Subject: [PATCH 158/171] implement radixtarget --- bbot/core/event/base.py | 6 ++- bbot/core/helpers/dns/dns.py | 20 ++++---- bbot/core/helpers/misc.py | 71 ++------------------------- bbot/scanner/target.py | 61 ++++++++++++----------- bbot/test/test_step_1/test_helpers.py | 10 ---- poetry.lock | 13 ++++- pyproject.toml | 1 + 7 files changed, 65 insertions(+), 117 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index c036b9618..8c69d829d 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -8,6 +8,7 @@ from datetime import datetime from contextlib import suppress from urllib.parse import urljoin +from radixtarget import RadixTarget from pydantic import BaseModel, field_validator from .helpers import * @@ -15,7 +16,6 @@ from bbot.core.helpers import ( extract_words, get_file_extension, - host_in_host, is_domain, is_subdomain, is_ip, @@ -580,7 +580,9 @@ def __contains__(self, other): if self.host == other.host: return True # hostnames and IPs - return host_in_host(other.host, self.host) + radixtarget = RadixTarget() + radixtarget.insert(self.host) + return bool(radixtarget.search(other.host)) return False def json(self, mode="json", siem_friendly=False): diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 9764687bf..2d78d2c19 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -2,9 +2,10 @@ import logging import dns.exception import dns.asyncresolver +from radixtarget import RadixTarget from bbot.core.engine import EngineClient -from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name, host_in_host +from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name from .engine import DNSEngine @@ -63,10 +64,9 @@ def __init__(self, parent_helper): # wildcard handling self.wildcard_disable = self.config.get("dns_wildcard_disable", False) - self.wildcard_ignore = self.config.get("dns_wildcard_ignore", None) - if not self.wildcard_ignore: - self.wildcard_ignore = [] - self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) + self.wildcard_ignore = RadixTarget() + for d in self.config.get("dns_wildcard_ignore", []): + self.wildcard_ignore.insert(d) # copy the system's current resolvers to a text file for tool use self.system_resolvers = dns.resolver.Resolver().nameservers @@ -150,10 +150,12 @@ def _wildcard_prevalidation(self, host): return False # skip check if the query's parent domain is excluded in the config - for d in self.wildcard_ignore: - if host_in_host(host, d): - log.debug(f"Skipping wildcard detection on {host} because it is excluded in the config") - return False + wildcard_ignore = self.wildcard_ignore.search(host) + if wildcard_ignore: + log.debug( + f"Skipping wildcard detection on {host} because it or its parent domai ({wildcard_ignore}) is excluded in the config" + ) + return False return host diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index d6e3238d1..a4378069d 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -637,7 +637,7 @@ def is_ip_type(i): >>> is_ip_type("192.168.1.0/24") False """ - return isinstance(i, ipaddress._BaseV4) or isinstance(i, ipaddress._BaseV6) + return ipaddress._IPAddressBase in i.__class__.__mro__ def make_ip_type(s): @@ -663,78 +663,17 @@ def make_ip_type(s): >>> make_ip_type("evilcorp.com") 'evilcorp.com' """ + if not s: + raise ValueError(f'Invalid hostname: "{s}"') # IP address with suppress(Exception): - return ipaddress.ip_address(str(s).strip()) + return ipaddress.ip_address(s) # IP network with suppress(Exception): - return ipaddress.ip_network(str(s).strip(), strict=False) + return ipaddress.ip_network(s, strict=False) return s -def host_in_host(host1, host2): - """ - Checks if host1 is included within host2, either as a subdomain, IP, or IP network. - Used for scope calculations/decisions within BBOT. - - Args: - host1 (str or ipaddress.IPv4Address or ipaddress.IPv6Address or ipaddress.IPv4Network or ipaddress.IPv6Network): - The host to check for inclusion within host2. - host2 (str or ipaddress.IPv4Address or ipaddress.IPv6Address or ipaddress.IPv4Network or ipaddress.IPv6Network): - The host within which to check for the inclusion of host1. - - Returns: - bool: True if host1 is included in host2, otherwise False. - - Examples: - >>> host_in_host("www.evilcorp.com", "evilcorp.com") - True - >>> host_in_host("evilcorp.com", "www.evilcorp.com") - False - >>> host_in_host(ipaddress.IPv6Address('dead::beef'), ipaddress.IPv6Network('dead::/64')) - True - >>> host_in_host(ipaddress.IPv4Address('192.168.1.1'), ipaddress.IPv4Network('10.0.0.0/8')) - False - - Notes: - - If checking an IP address/network, you MUST FIRST convert your IP into an ipaddress object (e.g. via `make_ip_type()`) before passing it to this function. - """ - - """ - Is host1 included in host2? - "www.evilcorp.com" in "evilcorp.com"? --> True - "evilcorp.com" in "www.evilcorp.com"? --> False - IPv6Address('dead::beef') in IPv6Network('dead::/64')? --> True - IPv4Address('192.168.1.1') in IPv4Network('10.0.0.0/8')? --> False - - Very important! Used throughout BBOT for scope calculations/decisions. - - Works with hostnames, IPs, and IP networks. - """ - - if not host1 or not host2: - return False - - # check if hosts are IP types - host1_ip_type = is_ip_type(host1) - host2_ip_type = is_ip_type(host2) - # if both hosts are IP types - if host1_ip_type and host2_ip_type: - if not host1.version == host2.version: - return False - host1_net = ipaddress.ip_network(host1) - host2_net = ipaddress.ip_network(host2) - return host1_net.subnet_of(host2_net) - - # else hostnames - elif not (host1_ip_type or host2_ip_type): - host2_len = len(host2.split(".")) - host1_truncated = ".".join(host1.split(".")[-host2_len:]) - return host1_truncated == host2 - - return False - - def sha1(data): """ Computes the SHA-1 hash of the given data. diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index a0f8130c8..1016fd3cf 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -1,12 +1,13 @@ import re +import copy import logging import ipaddress from contextlib import suppress +from radixtarget import RadixTarget from bbot.errors import * from bbot.modules.base import BaseModule from bbot.core.event import make_event, is_event -from bbot.core.helpers.misc import ip_network_parents, is_ip_type, domain_parents log = logging.getLogger("bbot.core.target") @@ -19,7 +20,8 @@ class Target: strict_scope (bool): Flag indicating whether to consider child domains in-scope. If set to True, only the exact hosts specified and not their children are considered part of the target. - _events (dict): Dictionary mapping hosts to events related to the target. + _radix (RadixTree): Radix tree for quick IP/DNS lookups. + _events (set): Flat set of contained events. Examples: Basic usage @@ -85,8 +87,9 @@ def __init__(self, *targets, strict_scope=False): "ORG_STUB": re.compile(r"^ORG:(.*)", re.IGNORECASE), "ASN": re.compile(r"^ASN:(.*)", re.IGNORECASE), } + self._events = set() + self._radix = RadixTarget() - self._events = dict() if len(targets) > 0: log.verbose(f"Creating events from {len(targets):,} targets") for t in targets: @@ -142,17 +145,18 @@ def add_target(self, t, event_type=None): if not str(t).startswith("#"): raise ValidationError(f'Could not add target "{t}": {e}') - try: - self._events[event.host].add(event) - except KeyError: - self._events[event.host] = { - event, - } + radix_data = self._radix.search(event.host) + if radix_data is None: + radix_data = {event} + self._radix.insert(event.host, radix_data) + else: + radix_data.add(event) + self._events.add(event) @property def events(self): """ - A generator property that yields all events in the target. + Returns all events in the target. Yields: Event object: One of the Event objects stored in the `_events` dictionary. @@ -164,14 +168,12 @@ def events(self): Notes: - This property is read-only. - - Iterating over this property gives you one event at a time from the `_events` dictionary. """ - for _events in self._events.values(): - yield from _events + return self._events def copy(self): """ - Creates and returns a copy of the Target object, including a shallow copy of the `_events` attribute. + Creates and returns a copy of the Target object, including a shallow copy of the `_events` and `_radix` attributes. Returns: Target: A new Target object with the sameattributes as the original. @@ -193,12 +195,13 @@ def copy(self): - The `scan` object reference is kept intact in the copied Target object. """ self_copy = self.__class__() - self_copy._events = dict(self._events) + self_copy._events = set(self._events) + self_copy._radix = copy.copy(self._radix) return self_copy def get(self, host): """ - Gets the event associated with the specified host from the target's `_events` dictionary. + Gets the event associated with the specified host from the target's radix tree. Args: host (Event, Target, or str): The hostname, IP, URL, or event to look for. @@ -224,15 +227,15 @@ def get(self, host): return if other.host: with suppress(KeyError, StopIteration): - return next(iter(self._events[other.host])) - if is_ip_type(other.host): - for n in ip_network_parents(other.host, include_self=True): - with suppress(KeyError, StopIteration): - return next(iter(self._events[n])) - elif not self.strict_scope: - for h in domain_parents(other.host): - with suppress(KeyError, StopIteration): - return next(iter(self._events[h])) + result = self._radix.search(other.host) + if result is not None: + for event in result: + # if the result is a dns name and strict scope is enabled + if isinstance(result, str) and self.strict_scope: + # if the result doesn't exactly equal the host, abort + if event.host != other.host: + return + return event def _contains(self, other): if self.get(other) is not None: @@ -282,11 +285,11 @@ def __len__(self): - For other types of hosts, each unique event is counted as one. """ num_hosts = 0 - for host, _events in self._events.items(): - if type(host) in (ipaddress.IPv4Network, ipaddress.IPv6Network): - num_hosts += host.num_addresses + for event in self._events: + if isinstance(event.host, (ipaddress.IPv4Network, ipaddress.IPv6Network)): + num_hosts += event.host.num_addresses else: - num_hosts += len(_events) + num_hosts += 1 return num_hosts diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 0ce3e0c76..4e3f3993e 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -103,16 +103,6 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): assert helpers.domain_stem("evilcorp.co.uk") == "evilcorp" assert helpers.domain_stem("www.evilcorp.co.uk") == "www.evilcorp" - assert helpers.host_in_host("www.evilcorp.com", "evilcorp.com") == True - assert helpers.host_in_host("asdf.www.evilcorp.com", "evilcorp.com") == True - assert helpers.host_in_host("evilcorp.com", "www.evilcorp.com") == False - assert helpers.host_in_host("evilcorp.com", "evilcorp.com") == True - assert helpers.host_in_host("evilcorp.com", "eevilcorp.com") == False - assert helpers.host_in_host("eevilcorp.com", "evilcorp.com") == False - assert helpers.host_in_host("evilcorp.com", "evilcorp") == False - assert helpers.host_in_host("evilcorp", "evilcorp.com") == False - assert helpers.host_in_host("evilcorp.com", "com") == True - assert tuple(await helpers.re.extract_emails("asdf@asdf.com\nT@t.Com&a=a@a.com__ b@b.com")) == ( "asdf@asdf.com", "t@t.com", diff --git a/poetry.lock b/poetry.lock index 05386f0a4..034b4fef6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2084,6 +2084,17 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "radixtarget" +version = "1.0.0.15" +description = "Check whether an IP address belongs to a cloud provider" +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "radixtarget-1.0.0.15-py3-none-any.whl", hash = "sha256:4e3f0620bfbc0ef2ff3d71270dd281c0e8428906d260f737f82b573a7b636dd8"}, + {file = "radixtarget-1.0.0.15.tar.gz", hash = "sha256:c8294ebbb76e6d2826deaa8fe18d568308eddfd25f20644e166c492d2626a70c"}, +] + [[package]] name = "regex" version = "2024.4.16" @@ -2625,4 +2636,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "15633b02fcedb3d044f4e40a45ce1e9dd7209608a0389175a4523e3810a8504b" +content-hash = "100618fdac0971d8b3662f2bfe72a8fae4f221ca78dfc6a0edf605859ab64f3f" diff --git a/pyproject.toml b/pyproject.toml index 0cc6eed31..1c0c15a9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ pyzmq = "^25.1.2" regex = "^2024.4.16" unidecode = "^1.3.8" cloudcheck = "^4.0.0.345" +radixtarget = "^1.0.0.15" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From af110c9a6bf4fec254471e63674d11beee1a30c9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:35:32 -0400 Subject: [PATCH 159/171] better scope tests --- bbot/scanner/target.py | 2 +- bbot/test/test_step_1/test_target.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 1016fd3cf..7059bda70 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -231,7 +231,7 @@ def get(self, host): if result is not None: for event in result: # if the result is a dns name and strict scope is enabled - if isinstance(result, str) and self.strict_scope: + if isinstance(event.host, str) and self.strict_scope: # if the result doesn't exactly equal the host, abort if event.host != other.host: return diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index dced8af02..521593191 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -38,3 +38,13 @@ def test_target(bbot_scanner): assert scan1.target.get("2001:4860:4860::888c") is None assert str(scan1.target.get("www.api.publicapis.org").host) == "api.publicapis.org" assert scan1.target.get("publicapis.org") is None + + from bbot.scanner.target import Target + target = Target("evilcorp.com") + assert not "com" in target + assert "evilcorp.com" in target + assert "www.evilcorp.com" in target + strict_target = Target("evilcorp.com", strict_scope=True) + assert not "com" in strict_target + assert "evilcorp.com" in strict_target + assert not "www.evilcorp.com" in strict_target From 8f72db74e982778b491d5a4bb2fef94fe7779a80 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:35:46 -0400 Subject: [PATCH 160/171] blacked --- bbot/test/test_step_1/test_target.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 521593191..cf210c0f6 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -40,6 +40,7 @@ def test_target(bbot_scanner): assert scan1.target.get("publicapis.org") is None from bbot.scanner.target import Target + target = Target("evilcorp.com") assert not "com" in target assert "evilcorp.com" in target From 4f073125ca89d1f523667f15b127d008148a5940 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:42:23 -0400 Subject: [PATCH 161/171] update cloudcheck --- poetry.lock | 9 +++++---- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 034b4fef6..be6fea410 100644 --- a/poetry.lock +++ b/poetry.lock @@ -388,18 +388,19 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "4.0.0.345" +version = "5.0.0.350" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-4.0.0.345-py3-none-any.whl", hash = "sha256:82a1cecaa0ec35a50d6c1e4884a9535eb4c1c788b845b0c4a91b44935f4dc765"}, - {file = "cloudcheck-4.0.0.345.tar.gz", hash = "sha256:787953a305c0be6e6eb4ceb9990dccb633f9e1429d5ebfda7acf7dca35b3caeb"}, + {file = "cloudcheck-5.0.0.350-py3-none-any.whl", hash = "sha256:6f2ed981818bde6d8b6c5a6413a843e11d0aa1a4bf8b36452dcae1030a537dd6"}, + {file = "cloudcheck-5.0.0.350.tar.gz", hash = "sha256:cb59dfef966268ebc176e242634b84a3423a84ffaf4fac40566f37edfaddc106"}, ] [package.dependencies] httpx = ">=0.26,<0.28" pydantic = ">=2.4.2,<3.0.0" +radixtarget = ">=1.0.0.14,<2.0.0.0" regex = ">=2024.4.16,<2025.0.0" [[package]] @@ -2636,4 +2637,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "100618fdac0971d8b3662f2bfe72a8fae4f221ca78dfc6a0edf605859ab64f3f" +content-hash = "ed8bb07e4ff5a5f665402db33f9016409547bef1ccb6a8c2c626c44fde075abb" diff --git a/pyproject.toml b/pyproject.toml index 1c0c15a9c..7ba00c488 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,8 @@ jinja2 = "^3.1.3" pyzmq = "^25.1.2" regex = "^2024.4.16" unidecode = "^1.3.8" -cloudcheck = "^4.0.0.345" radixtarget = "^1.0.0.15" +cloudcheck = "^5.0.0.350" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From 53f71e9af883396da0cb22712f306c5b3129f12c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:43:09 -0400 Subject: [PATCH 162/171] fix cloudcheck --- bbot/modules/internal/cloud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index e6bab4baa..29abef4d2 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -24,9 +24,9 @@ async def handle_event(self, event, kwargs): hosts_to_check = set(str(s) for s in event.resolved_hosts) hosts_to_check.add(str(event.host_original)) for host in hosts_to_check: - provider, provider_type, subnet = self.helpers.cloudcheck(host) - if provider: - event.add_tag(f"{provider_type}-{provider}") + for provider, provider_type, subnet in self.helpers.cloudcheck(host) + if provider: + event.add_tag(f"{provider_type}-{provider}") found = set() # look for cloud assets in hosts, http responses From a3c8e61da81c42decfc6de3659928d0cc6ddba9f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 17:48:53 -0400 Subject: [PATCH 163/171] better target tests --- bbot/modules/internal/cloud.py | 2 +- bbot/scanner/target.py | 25 ++++++++++++------------- bbot/test/test_step_1/test_target.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py index 29abef4d2..7939487fd 100644 --- a/bbot/modules/internal/cloud.py +++ b/bbot/modules/internal/cloud.py @@ -24,7 +24,7 @@ async def handle_event(self, event, kwargs): hosts_to_check = set(str(s) for s in event.resolved_hosts) hosts_to_check.add(str(event.host_original)) for host in hosts_to_check: - for provider, provider_type, subnet in self.helpers.cloudcheck(host) + for provider, provider_type, subnet in self.helpers.cloudcheck(host): if provider: event.add_tag(f"{provider_type}-{provider}") diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 7059bda70..b19d1b6a6 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -118,11 +118,8 @@ def add_target(self, t, event_type=None): t = [t] for single_target in t: if type(single_target) == self.__class__: - for k, v in single_target._events.items(): - try: - self._events[k].update(v) - except KeyError: - self._events[k] = set(single_target._events[k]) + for event in single_target.events: + self._add_event(event) else: if is_event(single_target): event = single_target @@ -144,14 +141,7 @@ def add_target(self, t, event_type=None): # allow commented lines if not str(t).startswith("#"): raise ValidationError(f'Could not add target "{t}": {e}') - - radix_data = self._radix.search(event.host) - if radix_data is None: - radix_data = {event} - self._radix.insert(event.host, radix_data) - else: - radix_data.add(event) - self._events.add(event) + self._add_event(event) @property def events(self): @@ -237,6 +227,15 @@ def get(self, host): return return event + def _add_event(self, event): + radix_data = self._radix.search(event.host) + if radix_data is None: + radix_data = {event} + self._radix.insert(event.host, radix_data) + else: + radix_data.add(event) + self._events.add(event) + def _contains(self, other): if self.get(other) is not None: return True diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index cf210c0f6..ed5c1b7ef 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -49,3 +49,14 @@ def test_target(bbot_scanner): assert not "com" in strict_target assert "evilcorp.com" in strict_target assert not "www.evilcorp.com" in strict_target + + target = Target() + target.add_target("evilcorp.com") + assert not "com" in target + assert "evilcorp.com" in target + assert "www.evilcorp.com" in target + strict_target = Target(strict_scope=True) + strict_target.add_target("evilcorp.com") + assert not "com" in strict_target + assert "evilcorp.com" in strict_target + assert not "www.evilcorp.com" in strict_target From d42c189bf4eef282cd6bcb3d7c8b41143455bdab Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 18:00:46 -0400 Subject: [PATCH 164/171] fix typo --- bbot/core/helpers/dns/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 2d78d2c19..5b5365f28 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -153,7 +153,7 @@ def _wildcard_prevalidation(self, host): wildcard_ignore = self.wildcard_ignore.search(host) if wildcard_ignore: log.debug( - f"Skipping wildcard detection on {host} because it or its parent domai ({wildcard_ignore}) is excluded in the config" + f"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config" ) return False From 8c07684ca7364966cf5b688e374ce6e2e4134415 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 18:18:28 -0400 Subject: [PATCH 165/171] better dns name sanitization --- bbot/core/helpers/dns/dns.py | 4 +--- bbot/core/helpers/dns/engine.py | 21 ++++++++++++++------- bbot/test/test_step_1/test_dns.py | 11 +++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 5b5365f28..7f775483c 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -152,9 +152,7 @@ def _wildcard_prevalidation(self, host): # skip check if the query's parent domain is excluded in the config wildcard_ignore = self.wildcard_ignore.search(host) if wildcard_ignore: - log.debug( - f"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config" - ) + log.debug(f"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config") return False return host diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index b8e184264..6018e0e3f 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -403,7 +403,8 @@ def new_task(query, rdtype): if queries: # Start a new task for each one completed, if URLs remain new_task(*queries.pop(0)) - def extract_targets(self, record): + @staticmethod + def extract_targets(record): """ Extracts hostnames or IP addresses from a given DNS record. @@ -429,24 +430,30 @@ def extract_targets(self, record): """ results = set() + + def add_result(rdtype, _record): + cleaned = clean_dns_record(_record) + if cleaned: + results.add((rdtype, cleaned)) + rdtype = str(record.rdtype.name).upper() if rdtype in ("A", "AAAA", "NS", "CNAME", "PTR"): - results.add((rdtype, clean_dns_record(record))) + add_result(rdtype, record) elif rdtype == "SOA": - results.add((rdtype, clean_dns_record(record.mname))) + add_result(rdtype, record.mname) elif rdtype == "MX": - results.add((rdtype, clean_dns_record(record.exchange))) + add_result(rdtype, record.exchange) elif rdtype == "SRV": - results.add((rdtype, clean_dns_record(record.target))) + add_result(rdtype, record.target) elif rdtype == "TXT": for s in record.strings: s = smart_decode(s) for match in dns_name_regex.finditer(s): start, end = match.span() host = s[start:end] - results.add((rdtype, host)) + add_result(rdtype, host) elif rdtype == "NSEC": - results.add((rdtype, clean_dns_record(record.next))) + add_result(rdtype, record.next) else: log.warning(f'Unknown DNS record type "{rdtype}"') return results diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index afc5c1967..05796e464 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -29,6 +29,17 @@ async def test_dns_engine(bbot_scanner): pass_2 = True assert pass_1 and pass_2 + from bbot.core.helpers.dns.engine import DNSEngine + from bbot.core.helpers.dns.mock import MockResolver + + # ensure dns records are being properly cleaned + mockresolver = MockResolver({"evilcorp.com": {"MX": ["0 ."]}}) + mx_records = await mockresolver.resolve("evilcorp.com", rdtype="MX") + results = set() + for r in mx_records: + results.update(DNSEngine.extract_targets(r)) + assert not results + @pytest.mark.asyncio async def test_dns_resolution(bbot_scanner): From e4fd60af06d6e9570105f7fdf49bd6c4d7d46661 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Apr 2024 19:06:51 -0400 Subject: [PATCH 166/171] fix ffuf tests --- .../test_step_2/module_tests/test_module_ffuf_shortnames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py b/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py index cbbec11ea..1f624a410 100644 --- a/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py +++ b/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py @@ -143,7 +143,7 @@ async def setup_after_prep(self, module_test): tags=["shortname-file"], ) ) - module_test.scan.target._events["http://127.0.0.1:8888"] = seed_events + module_test.scan.target._events = set(seed_events) expect_args = {"method": "GET", "uri": "/administrator.aspx"} respond_args = {"response_data": "alive"} From faf61eecd47c6ef88aa9b317ac80a8700b064439 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 23 Apr 2024 11:25:37 -0400 Subject: [PATCH 167/171] small scope tweak --- bbot/core/event/base.py | 2 +- bbot/scanner/manager.py | 1 - .../test_manager_scope_accuracy.py | 28 +++---- bbot/test/test_step_1/test_scope.py | 75 ++++++++++++++++++- 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 8c69d829d..d7eabd6db 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -93,7 +93,7 @@ class BaseEvent: # Always emit this event type even if it's not in scope _always_emit = False # Always emit events with these tags even if they're not in scope - _always_emit_tags = ["affiliate"] + _always_emit_tags = ["affiliate", "target"] # Bypass scope checking and dns resolution, distribute immediately to modules # This is useful for "end-of-line" events like FINDING and VULNERABILITY _quick_emit = False diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 76d7b6028..6fa59cf3e 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -43,7 +43,6 @@ async def init_events(self, events): sorted_events = sorted(events, key=lambda e: len(e.data)) for event in [self.scan.root_event] + sorted_events: event._dummy = False - event.scope_distance = 0 event.web_spider_distance = 0 event.scan = self.scan if event.source is None: diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index bc79a0029..dbca45276 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -750,12 +750,12 @@ def custom_setup(scan): "127.0.0.0/31", modules=["sslcert"], whitelist=["127.0.1.0"], - _config={"dns_resolution": False, "scope_report_distance": 0, "speculate": True, "modules": {"speculate": {"ports": "9999"}}}, + _config={"dns_resolution": False, "scope_report_distance": 0, "scope_search_distance": 1, "speculate": True, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) assert len(events) == 3 - assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) + assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) @@ -765,30 +765,30 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) assert len(all_events) == 11 - assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) - assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 1]) - assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1]) - assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 1]) + assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) + assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) assert len(all_events_nodups) == 9 - assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) - assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 1]) - assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1]) - assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 1]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): assert len(_graph_output_events) == 5 - assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) - assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index e51fec973..7435b82af 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -2,10 +2,58 @@ from ..test_step_2.module_tests.base import ModuleTestBase -class Scope_test_blacklist(ModuleTestBase): +class TestScopeBaseline(ModuleTestBase): targets = ["http://127.0.0.1:8888"] modules_overrides = ["httpx"] + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert len(events) == 6 + assert 1 == len( + [ + e + for e in events + if e.type == "URL_UNVERIFIED" + and str(e.host) == "127.0.0.1" + and e.scope_distance == 0 + and "target" in e.tags + ] + ) + # we have two of these because the host module considers "always_emit" in its outgoing deduplication + assert 2 == len( + [ + e + for e in events + if e.type == "IP_ADDRESS" + and e.data == "127.0.0.1" + and e.scope_distance == 0 + and str(e.module) == "host" + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "HTTP_RESPONSE" + and str(e.host) == "127.0.0.1" + and e.port == 8888 + and e.scope_distance == 0 + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "URL" and str(e.host) == "127.0.0.1" and e.port == 8888 and e.scope_distance == 0 + ] + ) + + +class TestScopeBlacklist(TestScopeBaseline): blacklist = ["127.0.0.1"] async def setup_after_prep(self, module_test): @@ -14,9 +62,32 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): + assert len(events) == 1 assert not any(e.type == "URL" for e in events) + assert not any(str(e.host) == "127.0.0.1" for e in events) -class Scope_test_whitelist(Scope_test_blacklist): +class TestScopeWhitelist(TestScopeBlacklist): blacklist = [] whitelist = ["255.255.255.255"] + + def check(self, module_test, events): + assert len(events) == 3 + assert not any(e.type == "URL" for e in events) + assert 1 == len( + [ + e + for e in events + if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.scope_distance == 1 and "target" in e.tags + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "URL_UNVERIFIED" + and str(e.host) == "127.0.0.1" + and e.scope_distance == 1 + and "target" in e.tags + ] + ) From e79d25537e603e4d810dd274f0088726a76a142e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 23 Apr 2024 11:50:21 -0400 Subject: [PATCH 168/171] Better debugging during scan cancellation --- bbot/core/helpers/misc.py | 2 ++ bbot/scanner/scanner.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 863cd1b10..b77aaa2b7 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1145,6 +1145,7 @@ def kill_children(parent_pid=None, sig=signal.SIGTERM): parent = psutil.Process(parent_pid) except psutil.NoSuchProcess: log.debug(f"No such PID: {parent_pid}") + return log.debug(f"Killing children of process ID {parent.pid}") children = parent.children(recursive=True) for child in children: @@ -1156,6 +1157,7 @@ def kill_children(parent_pid=None, sig=signal.SIGTERM): log.debug(f"No such PID: {child.pid}") except psutil.AccessDenied: log.debug(f"Error killing PID: {child.pid} - access denied") + log.debug(f"Finished killing children of process ID {parent.pid}") def str_or_file(s): diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 6af0f704c..cd58e7876 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -399,7 +399,13 @@ async def async_start(self): self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") finally: - self._cancel_tasks() + tasks = self._cancel_tasks() + self.debug(f"Awaiting {len(tasks):,} tasks") + for task in tasks: + self.debug(f"Awaiting {task}") + with contextlib.suppress(BaseException): + await task + self.debug(f"Awaited {len(tasks):,} tasks") await self._report() await self._cleanup() @@ -565,13 +571,14 @@ def stop(self): if not self._stopping: self._stopping = True self.status = "ABORTING" - self.hugewarning(f"Aborting scan") + self.hugewarning("Aborting scan") self.trace() self._cancel_tasks() self._drain_queues() self.helpers.kill_children() self._drain_queues() self.helpers.kill_children() + self.debug("Finished aborting scan") async def finish(self): """Finalizes the scan by invoking the `finished()` method on all active modules if new activity is detected. @@ -634,6 +641,7 @@ def _cancel_tasks(self): Returns: None """ + self.debug("Cancelling all scan tasks") tasks = [] # module workers for m in self.modules.values(): @@ -651,6 +659,8 @@ def _cancel_tasks(self): self.helpers.cancel_tasks_sync(tasks) # process pool self.process_pool.shutdown(cancel_futures=True) + self.debug("Finished cancelling all scan tasks") + return tasks async def _report(self): """Asynchronously executes the `report()` method for each module in the scan. From d158fa90fd8deaa0aec8f4da40aaf57787ea2359 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 23 Apr 2024 12:09:10 -0400 Subject: [PATCH 169/171] fix small cli bug and add tests for it --- bbot/scanner/preset/preset.py | 6 ++++-- bbot/test/test_step_1/test_cli.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 60dac85dd..57eb11a1a 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -381,8 +381,10 @@ def bake(self): for flag in baked_preset.flags: for module, preloaded in baked_preset.module_loader.preloaded().items(): module_flags = preloaded.get("flags", []) + module_type = preloaded.get("type", "scan") if flag in module_flags: - baked_preset.add_module(module, raise_error=False) + self.log_debug(f'Enabling module "{module}" because it has flag "{flag}"') + baked_preset.add_module(module, module_type, raise_error=False) # ensure we have output modules if not baked_preset.output_modules: @@ -433,7 +435,7 @@ def internal_modules(self): return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"] def add_module(self, module_name, module_type="scan", raise_error=True): - self.log_debug(f'Adding module "{module_name}"') + self.log_debug(f'Adding module "{module_name}" of type "{module_type}"') is_valid, reason, preloaded = self._is_valid_module(module_name, module_type, raise_error=raise_error) if not is_valid: self.log_debug(f'Unable to add {module_type} module "{module_name}": {reason}') diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 4a42b451a..a47ce665c 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -198,6 +198,13 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config): assert result == True assert "[ORG_STUB] evilcorp TARGET" in caplog.text + # activate modules by flag + caplog.clear() + assert not caplog.text + monkeypatch.setattr("sys.argv", ["bbot", "-f", "passive"]) + result = await cli._main() + assert result == True + # no args caplog.clear() assert not caplog.text From 630c87e5bf686034239e3ecdc1293e37acd5a0a8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 23 Apr 2024 12:24:56 -0400 Subject: [PATCH 170/171] remove resolved/unresolved tags as they are redundant --- bbot/modules/anubisdb.py | 2 +- bbot/modules/internal/dns.py | 4 +--- bbot/test/test_step_1/test_dns.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index 9864e3c6d..bf4c88e93 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -30,7 +30,7 @@ def abort_if_pre(self, hostname): async def abort_if(self, event): # abort if dns name is unresolved - if not "resolved" in event.tags: + if event.type == "DNS_NAME_UNRESOLVED": return True, "DNS name is unresolved" return await super().abort_if(event) diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index ea5e4efcf..b96b9b19c 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -94,9 +94,7 @@ async def handle_event(self, event, kwargs): if rdtype not in dns_children: dns_tags.add(f"{rdtype.lower()}-error") - if dns_children: - dns_tags.add("resolved") - elif not event_is_ip: + if not dns_children and not event_is_ip: dns_tags.add("unresolved") for rdtype, children in dns_children.items(): diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index afc5c1967..7fc93ab79 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -227,7 +227,6 @@ async def test_wildcards(bbot_scanner): "a-record", "target", "aaaa-wildcard", - "resolved", "in-scope", "subdomain", "aaaa-record", @@ -249,7 +248,7 @@ async def test_wildcards(bbot_scanner): for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" - and all(t in e.tags for t in ("a-record", "target", "resolved", "in-scope", "subdomain", "aaaa-record")) + and all(t in e.tags for t in ("a-record", "target", "in-scope", "subdomain", "aaaa-record")) and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard")) ] ) From c770d83acf77d6a6fb3bf4cf9bffb103fb7d4391 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 23 Apr 2024 12:53:57 -0400 Subject: [PATCH 171/171] better engine error handling during scan cancellation --- bbot/core/engine.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index c72eecbb3..24781ab3b 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -153,19 +153,26 @@ async def run_and_return(self, client_id, command_fn, **kwargs): error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}" trace = traceback.format_exc() result = {"_e": (error, trace)} - await self.socket.send_multipart([client_id, pickle.dumps(result)]) + await self.send_socket_multipart([client_id, pickle.dumps(result)]) async def run_and_yield(self, client_id, command_fn, **kwargs): self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({kwargs})") try: async for _ in command_fn(**kwargs): - await self.socket.send_multipart([client_id, pickle.dumps(_)]) - await self.socket.send_multipart([client_id, pickle.dumps({"_s": None})]) + await self.send_socket_multipart([client_id, pickle.dumps(_)]) + await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})]) except Exception as e: error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}" trace = traceback.format_exc() result = {"_e": (error, trace)} - await self.socket.send_multipart([client_id, pickle.dumps(result)]) + await self.send_socket_multipart([client_id, pickle.dumps(result)]) + + async def send_socket_multipart(self, *args, **kwargs): + try: + await self.socket.send_multipart(*args, **kwargs) + except Exception as e: + self.log.warning(f"Error sending ZMQ message: {e}") + self.log.trace(traceback.format_exc()) async def worker(self): try: