From 896036cdb3df1e0edbe67359cb5f1278654d5ccf Mon Sep 17 00:00:00 2001 From: Axylum Date: Tue, 14 Nov 2023 20:12:04 +0100 Subject: [PATCH] Formatting and numerous Python source code optimizations --- bbot/agent/agent.py | 25 +- bbot/cli.py | 5 +- bbot/core/configurator/args.py | 36 ++- bbot/core/configurator/environ.py | 54 +++-- bbot/core/configurator/files.py | 10 +- bbot/core/event/base.py | 181 +++++++------- bbot/core/event/helpers.py | 10 +- bbot/core/helpers/cache.py | 13 +- bbot/core/helpers/cloud/aws.py | 9 +- bbot/core/helpers/cloud/azure.py | 8 +- bbot/core/helpers/cloud/base.py | 57 +++-- bbot/core/helpers/cloud/digitalocean.py | 9 +- bbot/core/helpers/cloud/firebase.py | 6 +- bbot/core/helpers/cloud/gcp.py | 8 +- bbot/core/helpers/command.py | 36 ++- bbot/core/helpers/depsinstaller/installer.py | 122 +++++----- bbot/core/helpers/diff.py | 147 ++++++------ bbot/core/helpers/dns.py | 136 ++++++----- bbot/core/helpers/files.py | 3 +- bbot/core/helpers/helper.py | 5 +- bbot/core/helpers/interactsh.py | 9 +- bbot/core/helpers/logger.py | 7 +- bbot/core/helpers/misc.py | 204 +++++++--------- bbot/core/helpers/modules.py | 130 ++++++----- bbot/core/helpers/names_generator.py | 10 +- bbot/core/helpers/ntlm.py | 61 ++--- bbot/core/helpers/ratelimiter.py | 2 +- bbot/core/helpers/regexes.py | 37 ++- bbot/core/helpers/url.py | 10 +- bbot/core/helpers/validators.py | 15 +- bbot/core/helpers/web.py | 95 +++----- bbot/core/helpers/wordcloud.py | 39 ++-- bbot/core/logger/logger.py | 6 +- bbot/db/neo4j.py | 1 + bbot/modules/anubisdb.py | 9 +- bbot/modules/azure_tenant.py | 8 +- bbot/modules/badsecrets.py | 3 +- bbot/modules/base.py | 124 +++++----- bbot/modules/bevigil.py | 15 +- bbot/modules/bucket_aws.py | 13 +- bbot/modules/bucket_firebase.py | 4 +- bbot/modules/bucket_gcp.py | 2 +- bbot/modules/builtwith.py | 20 +- bbot/modules/bypass403.py | 18 +- bbot/modules/c99.py | 6 +- bbot/modules/censys.py | 11 +- bbot/modules/certspotter.py | 3 +- bbot/modules/chaos.py | 5 +- bbot/modules/columbus.py | 6 +- bbot/modules/crt.py | 8 +- bbot/modules/deadly/ffuf.py | 139 +++++------ bbot/modules/deadly/nuclei.py | 43 ++-- bbot/modules/deadly/vhost.py | 71 +++--- bbot/modules/dnszonetransfer.py | 12 +- bbot/modules/ffuf_shortnames.py | 219 +++++++++-------- bbot/modules/filedownload.py | 115 +++++---- bbot/modules/fullhunt.py | 3 +- bbot/modules/generic_ssrf.py | 32 +-- bbot/modules/git.py | 5 +- bbot/modules/gowitness.py | 11 +- bbot/modules/hackertarget.py | 3 +- bbot/modules/host_header.py | 5 +- bbot/modules/httpx.py | 24 +- bbot/modules/hunt.py | 3 +- bbot/modules/hunterio.py | 19 +- bbot/modules/iis_shortnames.py | 34 ++- bbot/modules/internal/excavate.py | 26 +-- bbot/modules/internal/speculate.py | 53 +++-- bbot/modules/ip2location.py | 6 +- bbot/modules/ipneighbor.py | 6 +- bbot/modules/ipstack.py | 6 +- bbot/modules/leakix.py | 9 +- bbot/modules/masscan.py | 8 +- bbot/modules/massdns.py | 38 ++- bbot/modules/myssl.py | 3 +- bbot/modules/nmap.py | 11 +- bbot/modules/nsec.py | 4 +- bbot/modules/ntlm.py | 29 ++- bbot/modules/oauth.py | 37 ++- bbot/modules/otx.py | 3 +- bbot/modules/output/asset_inventory.py | 28 +-- bbot/modules/output/base.py | 25 +- bbot/modules/output/discord.py | 32 ++- bbot/modules/output/http.py | 3 +- bbot/modules/output/web_report.py | 8 +- bbot/modules/output/websocket.py | 2 +- bbot/modules/paramminer_cookies.py | 6 +- bbot/modules/paramminer_getparams.py | 6 +- bbot/modules/paramminer_headers.py | 41 ++-- bbot/modules/pgp.py | 2 +- bbot/modules/rapiddns.py | 3 +- bbot/modules/report/asn.py | 30 +-- bbot/modules/riddler.py | 3 +- bbot/modules/robots.py | 6 +- bbot/modules/secretsdb.py | 8 +- bbot/modules/securitytrails.py | 3 +- bbot/modules/shodan_dns.py | 6 +- bbot/modules/sitedossier.py | 8 +- bbot/modules/skymem.py | 4 +- bbot/modules/smuggler.py | 1 - bbot/modules/sslcert.py | 22 +- bbot/modules/subdomain_hijack.py | 28 +-- bbot/modules/subdomaincenter.py | 5 +- bbot/modules/telerik.py | 10 +- bbot/modules/templates/subdomain_enum.py | 39 ++-- bbot/modules/threatminer.py | 3 +- bbot/modules/url_manipulation.py | 44 ++-- bbot/modules/urlscan.py | 16 +- bbot/modules/viewdns.py | 2 +- bbot/modules/virustotal.py | 6 +- bbot/modules/wafw00f.py | 27 ++- bbot/scanner/manager.py | 65 +++--- bbot/scanner/scanner.py | 220 +++++++++--------- bbot/scanner/stats.py | 10 +- bbot/scanner/target.py | 29 +-- bbot/scripts/docs.py | 15 +- bbot/test/bbot_fixtures.py | 7 +- bbot/test/conftest.py | 14 +- bbot/test/test_step_1/test_agent.py | 1 - bbot/test/test_step_1/test_cli.py | 6 +- bbot/test/test_step_1/test_command.py | 4 +- bbot/test/test_step_1/test_dns.py | 6 +- bbot/test/test_step_1/test_events.py | 35 +-- bbot/test/test_step_1/test_helpers.py | 80 ++++--- bbot/test/test_step_1/test_manager.py | 18 +- bbot/test/test_step_1/test_modules_basic.py | 25 +- bbot/test/test_step_1/test_python_api.py | 8 +- bbot/test/test_step_1/test_regexes.py | 28 +-- bbot/test/test_step_1/test_scan.py | 12 +- bbot/test/test_step_1/test_scope.py | 2 +- bbot/test/test_step_1/test_target.py | 25 +- bbot/test/test_step_1/test_web.py | 26 +-- bbot/test/test_step_2/module_tests/base.py | 13 +- .../module_tests/test_module_anubisdb.py | 7 +- .../module_tests/test_module_azure_realm.py | 2 +- .../module_tests/test_module_badsecrets.py | 30 +-- .../module_tests/test_module_bevigil.py | 9 +- .../module_tests/test_module_binaryedge.py | 4 +- .../module_tests/test_module_bucket_aws.py | 4 +- .../module_tests/test_module_builtwith.py | 4 +- .../module_tests/test_module_certspotter.py | 2 +- .../module_tests/test_module_columbus.py | 2 +- .../module_tests/test_module_credshed.py | 8 +- .../module_tests/test_module_crt.py | 2 +- .../module_tests/test_module_dehashed.py | 4 +- .../module_tests/test_module_digitorus.py | 2 +- .../module_tests/test_module_dnscommonsrv.py | 9 +- .../module_tests/test_module_dnsdumpster.py | 4 +- .../module_tests/test_module_excavate.py | 13 +- .../module_tests/test_module_filedownload.py | 26 ++- .../module_tests/test_module_generic_ssrf.py | 5 +- .../module_tests/test_module_host_header.py | 11 +- .../module_tests/test_module_httpx.py | 2 +- .../module_tests/test_module_ipneighbor.py | 2 +- .../module_tests/test_module_leakix.py | 4 +- .../module_tests/test_module_massdns.py | 2 +- .../module_tests/test_module_myssl.py | 2 +- .../module_tests/test_module_nmap.py | 2 +- .../module_tests/test_module_otx.py | 2 +- .../module_tests/test_module_rapiddns.py | 3 +- .../module_tests/test_module_riddler.py | 3 +- .../module_tests/test_module_robots.py | 4 +- .../module_tests/test_module_sitedossier.py | 5 +- .../module_tests/test_module_smuggler.py | 2 +- .../test_module_subdomaincenter.py | 7 +- .../module_tests/test_module_subdomains.py | 7 +- .../module_tests/test_module_sublist3r.py | 7 +- .../module_tests/test_module_telerik.py | 2 +- .../module_tests/test_module_wayback.py | 2 +- .../module_tests/test_module_web_report.py | 20 +- .../module_tests/test_module_zoomeye.py | 4 +- 171 files changed, 1929 insertions(+), 2115 deletions(-) diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py index 1c8debc1e..a01f1386c 100644 --- a/bbot/agent/agent.py +++ b/bbot/agent/agent.py @@ -34,10 +34,10 @@ def __init__(self, config): def setup(self): if not self.url: - log.error(f"Must specify agent_url") + log.error("Must specify agent_url") return False if not self.token: - log.error(f"Must specify agent_token") + log.error("Must specify agent_token") return False return True @@ -45,7 +45,7 @@ async def ws(self, rebuild=False): if self._ws is None or rebuild: kwargs = {"close_timeout": 0.5} if self.token: - kwargs.update({"extra_headers": {"Authorization": f"Bearer {self.token}"}}) + kwargs["extra_headers"] = {"Authorization": f"Bearer {self.token}"} verbs = ("Building", "Built") if rebuild: verbs = ("Rebuilding", "Rebuilt") @@ -74,10 +74,9 @@ async def start(self): message = json.loads(message) message = messages.Message(**message) - if message.command == "ping": - if self.scan is None: - await self.send({"conversation": str(message.conversation), "message_type": "pong"}) - continue + if message.command == "ping" and self.scan is None: + await self.send({"conversation": str(message.conversation), "message_type": "pong"}) + continue command_type = getattr(messages, message.command, None) if command_type is None: @@ -116,7 +115,15 @@ async def send(self, message): await asyncio.sleep(1) # rebuild = True - async def start_scan(self, scan_id, name=None, targets=[], modules=[], output_modules=[], config={}): + async def start_scan(self, scan_id, name=None, targets=None, modules=None, output_modules=None, config=None): + if targets is None: + targets = [] + if modules is None: + modules = [] + if output_modules is None: + output_modules = [] + if config is None: + config = {} async with self._scan_lock: if self.scan is None: log.success( @@ -139,7 +146,7 @@ async def start_scan(self, scan_id, name=None, targets=[], modules=[], output_mo ) self.task = asyncio.create_task(self._start_scan_task(scan)) - return {"success": f"Started scan", "scan_id": scan.id} + return {"success": "Started scan", "scan_id": scan.id} else: msg = f"Scan {self.scan.id} already in progress" log.warning(msg) diff --git a/bbot/cli.py b/bbot/cli.py index b2df0c9b9..b68fc8807 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -26,13 +26,10 @@ log = logging.getLogger("bbot.cli") sys.stdout.reconfigure(line_buffering=True) - log_level = get_log_level() - from . import config - err = False scan_name = "" @@ -201,7 +198,7 @@ async def _main(): modules = set(scanner._scan_modules) for m in scanner._scan_modules: flags = module_loader._preloaded.get(m, {}).get("flags", []) - if not all(f in flags for f in options.require_flags): + if any(f not in flags for f in options.require_flags): log.verbose( f"Removing {m} because it does not have the required flags: {'+'.join(options.require_flags)}" ) diff --git a/bbot/core/configurator/args.py b/bbot/core/configurator/args.py index 795242911..27ded0edf 100644 --- a/bbot/core/configurator/args.py +++ b/bbot/core/configurator/args.py @@ -46,8 +46,7 @@ def parse_args(self, *args, **kwargs): match_and_exit(m, output_module_choices, msg="output module") for f in set(ret.flags + ret.require_flags): if f not in flag_choices and not self._dummy: - if f not in flag_choices and not self._dummy: - match_and_exit(f, flag_choices, msg="flag") + match_and_exit(f, flag_choices, msg="flag") return ret @@ -104,13 +103,11 @@ def error(self, message): ), ] - epilog = "EXAMPLES\n" for example in (scan_examples, usage_examples): for title, description, command in example: epilog += f"\n {title}:\n {command}\n" - parser = BBOTArgumentParser( description="Bighuge BLS OSINT Tool", formatter_class=argparse.RawTextHelpFormatter, epilog=epilog ) @@ -143,9 +140,19 @@ def error(self, message): help=f'Modules to enable. Choices: {",".join(module_choices)}', metavar="MODULE", ) - modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") modules.add_argument( - "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" + "-l", + "--list-modules", + action="store_true", + help="List available modules.", + ) + modules.add_argument( + "-em", + "--exclude-modules", + nargs="+", + default=[], + help="Exclude these modules.", + metavar="MODULE", ) modules.add_argument( "-f", @@ -155,13 +162,18 @@ def error(self, message): help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}', metavar="FLAG", ) - modules.add_argument("-lf", "--list-flags", action="store_true", help=f"List available flags.") + modules.add_argument( + "-lf", + "--list-flags", + action="store_true", + help="List available flags.", + ) modules.add_argument( "-rf", "--require-flags", nargs="+", default=[], - help=f"Only enable modules with these flags (e.g. -rf passive)", + help="Only enable modules with these flags (e.g. -rf passive)", metavar="FLAG", ) modules.add_argument( @@ -169,7 +181,7 @@ def error(self, message): "--exclude-flags", nargs="+", default=[], - help=f"Disable modules with these flags. (e.g. -ef aggressive)", + help="Disable modules with these flags. (e.g. -ef aggressive)", metavar="FLAG", ) modules.add_argument( @@ -200,7 +212,9 @@ def error(self, message): scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") - scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") + scan.add_argument( + "--dry-run", action="store_true", help="Abort before executing scan" + ) scan.add_argument( "--current-config", action="store_true", @@ -222,12 +236,10 @@ def error(self, message): misc = p.add_argument_group(title="Misc") misc.add_argument("--version", action="store_true", help="show BBOT version and exit") - cli_options = None with suppress(Exception): cli_options = dummy_parser.parse_args() - cli_config = [] diff --git a/bbot/core/configurator/environ.py b/bbot/core/configurator/environ.py index 4358bb78d..6948ab585 100644 --- a/bbot/core/configurator/environ.py +++ b/bbot/core/configurator/environ.py @@ -7,7 +7,6 @@ from ...modules import module_loader from ..helpers.misc import cpu_architecture, os_platform, os_platform_friendly - # keep track of whether BBOT is being executed via the CLI cli_execution = False @@ -48,9 +47,9 @@ def add_to_path(v, k="PATH"): var_list = os.environ.get(k, "").split(":") deduped_var_list = [] for _ in var_list: - if not _ in deduped_var_list: + if _ not in deduped_var_list: deduped_var_list.append(_) - if not v in deduped_var_list: + if v not in deduped_var_list: deduped_var_list = [v] + deduped_var_list new_var_str = ":".join(deduped_var_list) os.environ[k] = new_var_str @@ -61,7 +60,7 @@ def prepare_environment(bbot_config): Sync config to OS environment variables """ # ensure bbot_home - if not "home" in bbot_config: + if "home" not in bbot_config: bbot_config["home"] = "~/.bbot" home = Path(bbot_config["home"]).expanduser().resolve() bbot_config["home"] = str(home) @@ -78,7 +77,7 @@ def prepare_environment(bbot_config): # ensure bbot_tools bbot_tools = home / "tools" os.environ["BBOT_TOOLS"] = str(bbot_tools) - if not str(bbot_tools) in os.environ.get("PATH", "").split(":"): + if str(bbot_tools) not in os.environ.get("PATH", "").split(":"): os.environ["PATH"] = f'{bbot_tools}:{os.environ.get("PATH", "").strip(":")}' # ensure bbot_cache bbot_cache = home / "cache" @@ -99,15 +98,7 @@ def prepare_environment(bbot_config): # exchange certain options between CLI args and config if cli_execution and args.cli_options is not None: - # deps - bbot_config["retry_deps"] = args.cli_options.retry_deps - bbot_config["force_deps"] = args.cli_options.force_deps - bbot_config["no_deps"] = args.cli_options.no_deps - bbot_config["ignore_failed_deps"] = args.cli_options.ignore_failed_deps - # debug - bbot_config["debug"] = args.cli_options.debug - bbot_config["silent"] = args.cli_options.silent - + configure_cli_options(bbot_config) import logging log = logging.getLogger() @@ -122,11 +113,9 @@ def prepare_environment(bbot_config): # copy config to environment bbot_environ = flatten_config(bbot_config) - os.environ.update(bbot_environ) + os.environ |= bbot_environ - # handle HTTP proxy - http_proxy = bbot_config.get("http_proxy", "") - if http_proxy: + if http_proxy := bbot_config.get("http_proxy", ""): os.environ["HTTP_PROXY"] = http_proxy os.environ["HTTPS_PROXY"] = http_proxy else: @@ -142,12 +131,27 @@ def prepare_environment(bbot_config): urllib3.disable_warnings() ssl_verify = bbot_config.get("ssl_verify", False) if not ssl_verify: - import requests - import functools + disable_ssl_verification() + return bbot_config - requests.adapters.BaseAdapter.send = functools.partialmethod(requests.adapters.BaseAdapter.send, verify=False) - requests.adapters.HTTPAdapter.send = functools.partialmethod(requests.adapters.HTTPAdapter.send, verify=False) - requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) - requests.request = functools.partial(requests.request, verify=False) - return bbot_config +def configure_cli_options(bbot_config): + # deps + bbot_config["retry_deps"] = args.cli_options.retry_deps + bbot_config["force_deps"] = args.cli_options.force_deps + bbot_config["no_deps"] = args.cli_options.no_deps + bbot_config["ignore_failed_deps"] = args.cli_options.ignore_failed_deps + # debug + bbot_config["debug"] = args.cli_options.debug + bbot_config["silent"] = args.cli_options.silent + + +# TODO Rename this here and in `prepare_environment` +def disable_ssl_verification(): + import requests + import functools + + requests.adapters.BaseAdapter.send = functools.partialmethod(requests.adapters.BaseAdapter.send, verify=False) + requests.adapters.HTTPAdapter.send = functools.partialmethod(requests.adapters.HTTPAdapter.send, verify=False) + requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) + requests.request = functools.partial(requests.request, verify=False) diff --git a/bbot/core/configurator/files.py b/bbot/core/configurator/files.py index 16557d1ff..5433a3217 100644 --- a/bbot/core/configurator/files.py +++ b/bbot/core/configurator/files.py @@ -15,9 +15,11 @@ def _get_config(filename, name="config"): - notify = False - if sys.argv and sys.argv[0].endswith("bbot") and not any(x in sys.argv for x in ("-s", "--silent")): - notify = True + notify = bool( + sys.argv + and sys.argv[0].endswith("bbot") + and all(x not in sys.argv for x in ("-s", "--silent")) + ) filename = Path(filename).resolve() try: conf = OmegaConf.load(str(filename)) @@ -26,7 +28,7 @@ def _get_config(filename, name="config"): return conf except Exception as e: if filename.exists(): - raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") + raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") from e return OmegaConf.create() diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index c24601fef..0be5a40e6 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,15 +1,11 @@ -import json import asyncio -import logging -import ipaddress +import json import traceback +from datetime import datetime, timezone from typing import Optional -from datetime import datetime -from contextlib import suppress + from pydantic import BaseModel, validator -from .helpers import * -from bbot.core.errors import * from bbot.core.helpers import ( extract_words, split_host_port, @@ -21,12 +17,11 @@ domain_stem, make_netloc, make_ip_type, - smart_decode, get_file_extension, validators, tagify, ) - +from .helpers import * log = logging.getLogger("bbot.core.event") @@ -98,18 +93,18 @@ class BaseEvent: _data_validator = None def __init__( - self, - data, - event_type, - source=None, - module=None, - scan=None, - scans=None, - tags=None, - confidence=5, - timestamp=None, - _dummy=False, - _internal=None, + self, + data, + event_type, + source=None, + module=None, + scan=None, + scans=None, + tags=None, + confidence=5, + timestamp=None, + _dummy=False, + _internal=None, ): """ Initializes an Event object with the given parameters. @@ -149,11 +144,11 @@ def __init__( # keep track of whether this event has been recorded by the scan self._stats_recorded = False - self.timestamp = datetime.utcnow() + self.timestamp = datetime.now(timezone.utc) self._tags = set() if tags is not None: - self._tags = set(tagify(s) for s in tags) + self._tags = {tagify(s) for s in tags} self._data = None self._type = event_type @@ -167,7 +162,7 @@ def __init__( # self.scan holds the instantiated scan object (for helpers, etc.) self.scan = scan if (not self.scan) and (not self._dummy): - raise ValidationError(f"Must specify scan") + raise ValidationError("Must specify scan") # self.scans holds a list of scan IDs from scans that encountered this event self.scans = [] if scans is not None: @@ -184,7 +179,9 @@ def __init__( self.data = self._sanitize_data(data) except Exception as e: log.trace(traceback.format_exc()) - raise ValidationError(f'Error sanitizing event data "{data}" for type "{self.type}": {e}') + raise ValidationError( + f'Error sanitizing event data "{data}" for type "{self.type}": {e}' + ) from e if not self.data: raise ValidationError(f'Invalid event data "{data}" for type "{self.type}"') @@ -193,13 +190,11 @@ def __init__( self._source_id = None self.source = source if (not self.source) and (not self._dummy): - raise ValidationError(f"Must specify event source") + raise ValidationError("Must specify event source") # internal events are not ingested by output modules - if not self._dummy: - # removed this second part because it was making certain sslcert events internal - if _internal: # or source._internal: - self.make_internal() + if not self._dummy and _internal: + self.make_internal() # an event indicating whether the event has undergone DNS resolution self._resolved = asyncio.Event() @@ -210,11 +205,13 @@ def data(self): @property def resolved_hosts(self): - if is_ip(self.host): - return { + return ( + { self.host, } - return self._resolved_hosts + if is_ip(self.host) + else self._resolved_hosts + ) @data.setter def data(self, data): @@ -281,7 +278,7 @@ def tags(self): def tags(self, tags): if isinstance(tags, str): tags = (tags,) - self._tags = set(tagify(s) for s in tags) + self._tags = {tagify(s) for s in tags} def add_tag(self, tag): self._tags.add(tagify(tag)) @@ -319,19 +316,20 @@ def scope_distance(self, scope_distance): Note: The method will automatically update the relevant 'distance-' tags associated with the event. """ - if scope_distance >= 0: - new_scope_distance = None - # ensure scope distance does not increase (only allow setting to smaller values) - if self.scope_distance == -1: - new_scope_distance = scope_distance - else: - new_scope_distance = min(self.scope_distance, scope_distance) - if self._scope_distance != new_scope_distance: - self._scope_distance = new_scope_distance - for t in list(self.tags): - if t.startswith("distance-"): - self.remove_tag(t) - self.add_tag(f"distance-{new_scope_distance}") + if scope_distance < 0: + return + new_scope_distance = None + # ensure scope distance does not increase (only allow setting to smaller values) + if self.scope_distance == -1: + new_scope_distance = scope_distance + else: + new_scope_distance = min(self.scope_distance, scope_distance) + if self._scope_distance != new_scope_distance: + self._scope_distance = new_scope_distance + for t in list(self.tags): + if t.startswith("distance-"): + self.remove_tag(t) + self.add_tag(f"distance-{new_scope_distance}") @property def source(self): @@ -366,9 +364,7 @@ def source(self, source): @property def source_id(self): source_id = getattr(self.get_source(), "id", None) - if source_id is not None: - return source_id - return self._source_id + return source_id if source_id is not None else self._source_id def get_source(self): """ @@ -382,10 +378,7 @@ def get_sources(self, omit=False): sources = [] e = self while 1: - if omit: - source = e.get_source() - else: - source = e.source + source = e.get_source() if omit else e.source if e == source: break sources.append(source) @@ -586,10 +579,7 @@ def __contains__(self, other): return True # if hosts match if self.host and other.host: - if self.host == other.host: - return True - # hostnames and IPs - return host_in_host(other.host, self.host) + return True if self.host == other.host else host_in_host(other.host, self.host) return False def json(self, mode="json"): @@ -605,16 +595,12 @@ def json(self, mode="json"): Returns: dict: JSON-serializable dictionary representation of the event object. """ - j = dict() + j = {} for i in ("type", "id"): - v = getattr(self, i, "") - if v: - j.update({i: v}) + if v := getattr(self, i, ""): + j[i] = v data_attr = getattr(self, f"data_{mode}", None) - if data_attr is not None: - j["data"] = data_attr - else: - j["data"] = smart_decode(self.data) + j["data"] = data_attr if data_attr is not None else smart_decode(self.data) web_spider_distance = getattr(self, "web_spider_distance", None) if web_spider_distance is not None: j["web_spider_distance"] = web_spider_distance @@ -624,15 +610,14 @@ def json(self, mode="json"): j["timestamp"] = self.timestamp.timestamp() if self.host: j["resolved_hosts"] = [str(h) for h in self.resolved_hosts] - source_id = self.source_id - if source_id: + if source_id := self.source_id: j["source"] = source_id if self.tags: - j.update({"tags": list(self.tags)}) + j["tags"] = list(self.tags) if self.module: - j.update({"module": str(self.module)}) + j["module"] = str(self.module) if self.module_sequence: - j.update({"module_sequence": str(self.module_sequence)}) + j["module_sequence"] = str(self.module_sequence) # normalize non-primitive python objects for k, v in list(j.items()): @@ -770,8 +755,7 @@ def sanitize_data(self, data): class DictEvent(BaseEvent): def sanitize_data(self, data): - url = data.get("url", "") - if url: + if url := data.get("url", ""): self.parsed = validators.validate_url_parsed(url) return data @@ -779,19 +763,16 @@ def _data_human(self): return json.dumps(self.data, sort_keys=True) def _data_load(self, data): - if isinstance(data, str): - return json.loads(data) - return data + return json.loads(data) if isinstance(data, str) else data class DictHostEvent(DictEvent): def _host(self): if isinstance(self.data, dict) and "host" in self.data: return make_ip_type(self.data["host"]) - else: - parsed = getattr(self, "parsed") - if parsed is not None: - return make_ip_type(parsed.hostname) + parsed = getattr(self, "parsed") + if parsed is not None: + return make_ip_type(parsed.hostname) class ASN(DictEvent): @@ -870,8 +851,7 @@ def _host(self): def _words(self): stem = self.host_stem if not is_ptr(stem): - split_stem = stem.split(".") - if split_stem: + if split_stem := stem.split("."): leftmost_segment = split_stem[0] if leftmost_segment == "_wildcard": stem = ".".join(split_stem[1:]) @@ -925,8 +905,7 @@ def sanitize_data(self, data): if _url_extension_httpx_only: url_extension_httpx_only = [e.lower() for e in _url_extension_httpx_only] - extension = get_file_extension(parsed_path_lower) - if extension: + if extension := get_file_extension(parsed_path_lower): self.add_tag(f"extension-{extension}") if extension in url_extension_blacklist: self.add_tag("blacklisted") @@ -943,9 +922,7 @@ def with_port(self): def _words(self): first_elem = self.parsed.path.lstrip("/").split("/")[0] - if not "." in first_elem: - return extract_words(first_elem) - return set() + return extract_words(first_elem) if "." not in first_elem else set() def _host(self): return make_ip_type(self.parsed.hostname) @@ -954,7 +931,7 @@ def _data_id(self): # consider spider-danger tag when deduping data = super()._data_id() if "spider-danger" in self.tags: - data = "spider-danger" + data + data = f"spider-danger{data}" return data @@ -1026,7 +1003,7 @@ def sanitize_data(self, data): # move URL to the front of the dictionary for visibility data = dict(data) new_data = {"url": data.pop("url")} - new_data.update(data) + new_data |= data return new_data @@ -1169,16 +1146,16 @@ def _pretty_string(self): def make_event( - data, - event_type=None, - source=None, - module=None, - scan=None, - scans=None, - tags=None, - confidence=5, - dummy=False, - internal=None, + data, + event_type=None, + source=None, + module=None, + scan=None, + scans=None, + tags=None, + confidence=5, + dummy=False, + internal=None, ): """ Creates and returns a new event object or modifies an existing one. @@ -1260,7 +1237,9 @@ def make_event( data = validators.validate_host(data) except Exception as e: log.trace(traceback.format_exc()) - raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}') + raise ValidationError( + f'Error sanitizing event data "{data}" for type "{event_type}": {e}' + ) from e data_is_ip = is_ip(data) if event_type == "DNS_NAME" and data_is_ip: event_type = "IP_ADDRESS" @@ -1322,7 +1301,7 @@ def event_from_json(j): event._source_id = source_id return event except KeyError as e: - raise ValidationError(f"Event missing required field: {e}") + raise ValidationError(f"Event missing required field: {e}") from e def is_event(e): diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py index 2b3164bef..a0b740e4e 100644 --- a/bbot/core/event/helpers.py +++ b/bbot/core/event/helpers.py @@ -6,7 +6,6 @@ from bbot.core.helpers import sha1, smart_decode, smart_encode_punycode from bbot.core.helpers.regexes import event_type_regexes, event_id_regex - log = logging.getLogger("bbot.core.event.helpers") @@ -45,17 +44,12 @@ def get_event_type(data): for t, regexes in event_type_regexes.items(): for r in regexes: if r.match(data): - if t == "URL": - return "URL_UNVERIFIED", data - return t, data - + return ("URL_UNVERIFIED", data) if t == "URL" else (t, data) raise ValidationError(f'Unable to autodetect event type from "{data}"') def is_event_id(s): - if event_id_regex.match(str(s)): - return True - return False + return bool(event_id_regex.match(str(s))) def make_event_id(data, event_type): diff --git a/bbot/core/helpers/cache.py b/bbot/core/helpers/cache.py index 3eb54daf7..2ebfeb596 100644 --- a/bbot/core/helpers/cache.py +++ b/bbot/core/helpers/cache.py @@ -16,11 +16,10 @@ def cache_get(self, key, text=True, cache_hrs=24 * 7): """ filename = self.cache_filename(key) if filename.is_file(): - valid = self.is_cached(key, cache_hrs) - if valid: + if valid := self.is_cached(key, cache_hrs): open_kwargs = {} if text: - open_kwargs.update({"mode": "r", "encoding": "utf-8", "errors": "ignore"}) + open_kwargs |= {"mode": "r", "encoding": "utf-8", "errors": "ignore"} else: open_kwargs["mode"] = "rb" log.debug(f'Using cached content for "{key}"') @@ -93,10 +92,8 @@ def _truncate(self): if not self or len(self) <= self._max_size: return for nh in list(self._cache.keys()): - try: + with suppress(KeyError): del self._cache[nh] - except KeyError: - pass if not self or len(self) <= self._max_size: break @@ -113,9 +110,7 @@ def clear(self): return self._cache.clear() def _hash(self, v): - if type(v) == int: - return v - return hash(str(v)) + return v if type(v) == int else hash(str(v)) def __contains__(self, item): return self._hash(item) in self._cache diff --git a/bbot/core/helpers/cloud/aws.py b/bbot/core/helpers/cloud/aws.py index 9a2d7c518..de79425c6 100644 --- a/bbot/core/helpers/cloud/aws.py +++ b/bbot/core/helpers/cloud/aws.py @@ -1,6 +1,8 @@ from .base import BaseCloudProvider + + class AWS(BaseCloudProvider): domains = [ "amazon-dss.com", @@ -17,4 +19,9 @@ class AWS(BaseCloudProvider): "elasticbeanstalk.com", ] bucket_name_regex = r"[a-z0-9_][a-z0-9-\.]{1,61}[a-z0-9]" - regexes = {"STORAGE_BUCKET": [r"(" + bucket_name_regex + r")\.(s3-?(?:[a-z0-9-]*\.){1,2}amazonaws\.com)"]} + regexes = { + "STORAGE_BUCKET": [ + f"({bucket_name_regex}" + + r")\.(s3-?(?:[a-z0-9-]*\.){1,2}amazonaws\.com)" + ] + } diff --git a/bbot/core/helpers/cloud/azure.py b/bbot/core/helpers/cloud/azure.py index 83ceb5b03..852ab7383 100644 --- a/bbot/core/helpers/cloud/azure.py +++ b/bbot/core/helpers/cloud/azure.py @@ -1,6 +1,8 @@ from .base import BaseCloudProvider + + class Azure(BaseCloudProvider): # mostly pulled from https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure domains = [ @@ -46,4 +48,8 @@ class Azure(BaseCloudProvider): ] bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]" - regexes = {"STORAGE_BUCKET": [r"(" + bucket_name_regex + r")\.(blob\.core\.windows\.net)"]} + regexes = { + "STORAGE_BUCKET": [ + f"({bucket_name_regex}" + r")\.(blob\.core\.windows\.net)" + ] + } diff --git a/bbot/core/helpers/cloud/base.py b/bbot/core/helpers/cloud/base.py index cb1377d1d..120b02c82 100644 --- a/bbot/core/helpers/cloud/base.py +++ b/bbot/core/helpers/cloud/base.py @@ -12,11 +12,13 @@ def __init__(self, parent_helper): self.parent_helper = parent_helper self.name = str(self.__class__.__name__).lower() self.dummy_module = self.parent_helper._make_dummy_module(f"{self.name}_cloud", _type="scan") - self.bucket_name_regex = re.compile("^" + self.bucket_name_regex + "$", re.I) + self.bucket_name_regex = re.compile(f"^{self.bucket_name_regex}$", re.I) self.signatures = {} self.domain_regexes = [] - for domain in self.domains: - self.domain_regexes.append(re.compile(r"^(?:[\w\-]+\.)*" + rf"{re.escape(domain)}$")) + self.domain_regexes.extend( + re.compile(r"^(?:[\w\-]+\.)*" + rf"{re.escape(domain)}$") + for domain in self.domains + ) for event_type, regexes in self.regexes.items(): self.signatures[event_type] = [re.compile(r, re.I) for r in regexes] @@ -34,7 +36,7 @@ def excavate(self, event, http_body): for match in sig.findall(http_body): kwargs = dict(base_kwargs) kwargs["event_type"] = event_type - if not match in found: + if match not in found: found.add(match) if event_type == "STORAGE_BUCKET": self.emit_bucket(match, **kwargs) @@ -49,11 +51,10 @@ def speculate(self, event): for event_type, sigs in self.signatures.items(): found = set() for sig in sigs: - match = sig.match(event.data) - if match: + if match := sig.match(event.data): kwargs = dict(base_kwargs) kwargs["event_type"] = event_type - if not event.data in found: + if event.data not in found: found.add(event.data) if event_type == "STORAGE_BUCKET": self.emit_bucket(match.groups(), **kwargs) @@ -66,10 +67,10 @@ def emit_bucket(self, match, **kwargs): self.emit_event(**kwargs) def emit_event(self, *args, **kwargs): - excavate_module = self.parent_helper.scan.modules.get("excavate", None) - if excavate_module: - event = self.dummy_module.make_event(*args, **kwargs) - if event: + if excavate_module := self.parent_helper.scan.modules.get( + "excavate", None + ): + if event := self.dummy_module.make_event(*args, **kwargs): excavate_module.emit_event(event) def is_valid_bucket(self, bucket_name): @@ -77,23 +78,21 @@ def is_valid_bucket(self, bucket_name): def tag_event(self, event): # tag the event if - if event.host: - # its host directly matches this cloud provider's domains - if isinstance(event.host, str) and self.domain_match(event.host): - event.tags.update(self.base_tags) - # tag as buckets, etc. - for event_type, sigs in self.signatures.items(): - for sig in sigs: - if sig.match(event.host): - event.add_tag(f"cloud-{event_type}") - else: - # or it has a CNAME that matches this cloud provider's domains - for rh in event.resolved_hosts: - if not self.parent_helper.is_ip(rh) and self.domain_match(rh): - event.tags.update(self.base_tags) + if not event.host: + return + # its host directly matches this cloud provider's domains + if isinstance(event.host, str) and self.domain_match(event.host): + event.tags.update(self.base_tags) + # tag as buckets, etc. + for event_type, sigs in self.signatures.items(): + for sig in sigs: + if sig.match(event.host): + event.add_tag(f"cloud-{event_type}") + else: + # or it has a CNAME that matches this cloud provider's domains + for rh in event.resolved_hosts: + if not self.parent_helper.is_ip(rh) and self.domain_match(rh): + event.tags.update(self.base_tags) def domain_match(self, s): - for r in self.domain_regexes: - if r.match(s): - return True - return False + return any(r.match(s) for r in self.domain_regexes) diff --git a/bbot/core/helpers/cloud/digitalocean.py b/bbot/core/helpers/cloud/digitalocean.py index ffaae29f9..16145fc76 100644 --- a/bbot/core/helpers/cloud/digitalocean.py +++ b/bbot/core/helpers/cloud/digitalocean.py @@ -1,10 +1,17 @@ from .base import BaseCloudProvider + + class DigitalOcean(BaseCloudProvider): domains = [ "digitaloceanspaces.com", ] bucket_name_regex = r"[a-z0-9][a-z0-9-]{2,62}" - regexes = {"STORAGE_BUCKET": [r"(" + bucket_name_regex + r")\.([a-z]{3}[\d]{1}\.digitaloceanspaces\.com)"]} + regexes = { + "STORAGE_BUCKET": [ + f"({bucket_name_regex}" + + r")\.([a-z]{3}[\d]{1}\.digitaloceanspaces\.com)" + ] + } diff --git a/bbot/core/helpers/cloud/firebase.py b/bbot/core/helpers/cloud/firebase.py index f96648680..86f9fcace 100644 --- a/bbot/core/helpers/cloud/firebase.py +++ b/bbot/core/helpers/cloud/firebase.py @@ -1,10 +1,14 @@ from .base import BaseCloudProvider + + class Firebase(BaseCloudProvider): domains = [ "firebaseio.com", ] bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]" - regexes = {"STORAGE_BUCKET": [r"(" + bucket_name_regex + r")\.(firebaseio\.com)"]} + regexes = { + "STORAGE_BUCKET": [f"({bucket_name_regex}" + r")\.(firebaseio\.com)"] + } diff --git a/bbot/core/helpers/cloud/gcp.py b/bbot/core/helpers/cloud/gcp.py index 75959ad4e..818263281 100644 --- a/bbot/core/helpers/cloud/gcp.py +++ b/bbot/core/helpers/cloud/gcp.py @@ -1,6 +1,8 @@ from .base import BaseCloudProvider + + class GCP(BaseCloudProvider): domains = [ "googleapis.cn", @@ -10,4 +12,8 @@ class GCP(BaseCloudProvider): ] bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]" - regexes = {"STORAGE_BUCKET": [r"(" + bucket_name_regex + r")\.(storage\.googleapis\.com)"]} + regexes = { + "STORAGE_BUCKET": [ + f"({bucket_name_regex}" + r")\.(storage\.googleapis\.com)" + ] + } diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index bc28cbc82..359b58c29 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -98,12 +98,7 @@ async def run_live(self, *command, check=False, text=True, **kwargs): continue if not line: break - if text: - line = smart_decode(line).rstrip("\r\n") - else: - line = line.rstrip(b"\r\n") - yield line - + yield smart_decode(line).rstrip("\r\n") if text else line.rstrip(b"\r\n") if input_task is not None: try: await input_task @@ -178,17 +173,18 @@ async def _write_stdin(proc, _input): proc (subprocess.Popen): An active subprocess object. _input (str, bytes, list, tuple, async generator): The data to write to stdin. """ - if _input is not None: - if isinstance(_input, (str, bytes)): - _input = [_input] - if isinstance(_input, (list, tuple)): - for chunk in _input: - proc.stdin.write(smart_encode(chunk) + b"\n") - else: - async for chunk in _input: - proc.stdin.write(smart_encode(chunk) + b"\n") - await proc.stdin.drain() - proc.stdin.close() + if _input is None: + return + if isinstance(_input, (str, bytes)): + _input = [_input] + if isinstance(_input, (list, tuple)): + for chunk in _input: + proc.stdin.write(smart_encode(chunk) + b"\n") + else: + async for chunk in _input: + proc.stdin.write(smart_encode(chunk) + b"\n") + await proc.stdin.drain() + proc.stdin.close() def _prepare_command_kwargs(self, command, kwargs): @@ -215,11 +211,11 @@ def _prepare_command_kwargs(self, command, kwargs): (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)}) """ # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout) - if not "limit" in kwargs: + if "limit" not in kwargs: kwargs["limit"] = 1024 * 1024 * 100 - if not "stdout" in kwargs: + if "stdout" not in kwargs: kwargs["stdout"] = asyncio.subprocess.PIPE - if not "stderr" in kwargs: + if "stderr" not in kwargs: kwargs["stderr"] = asyncio.subprocess.PIPE sudo = kwargs.pop("sudo", False) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 72f6b5575..b62228e80 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -48,10 +48,7 @@ def __init__(self, parent_helper): self.force_deps = self.parent_helper.config.get("force_deps", False) self.retry_deps = self.parent_helper.config.get("retry_deps", False) self.ignore_failed_deps = self.parent_helper.config.get("ignore_failed_deps", False) - self.venv = "" - if sys.prefix != sys.base_prefix: - self.venv = sys.prefix - + self.venv = sys.prefix if sys.prefix != sys.base_prefix else "" self.all_modules_preloaded = module_loader.preloaded() self.ensure_root_lock = Lock() @@ -86,35 +83,34 @@ async def install(self, *modules): if len(dependencies) <= 0: log.debug(f'No setup to do for module "{m}"') succeeded.append(m) - continue - else: - if success is None or (success is False and self.retry_deps) or self.force_deps: - if not notified: - log.hugeinfo(f"Installing module dependencies. Please be patient, this may take a while.") - notified = True - log.verbose(f'Installing dependencies for module "{m}"') - # get sudo access if we need it - if preloaded.get("sudo", False) == True: - self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.') - success = await self.install_module(m) - self.setup_status[module_hash] = success - if success or self.ignore_failed_deps: - log.debug(f'Setup succeeded for module "{m}"') - succeeded.append(m) - else: - log.warning(f'Setup failed for module "{m}"') - failed.append(m) + elif success is None or (success is False and self.retry_deps) or self.force_deps: + if not notified: + log.hugeinfo( + "Installing module dependencies. Please be patient, this may take a while." + ) + notified = True + log.verbose(f'Installing dependencies for module "{m}"') + # get sudo access if we need it + if preloaded.get("sudo", False) == True: + self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.') + success = await self.install_module(m) + self.setup_status[module_hash] = success + if success or self.ignore_failed_deps: + log.debug(f'Setup succeeded for module "{m}"') + succeeded.append(m) else: - if success or self.ignore_failed_deps: - log.debug( - f'Skipping dependency install for module "{m}" because it\'s already done (--force-deps to re-run)' - ) - succeeded.append(m) - else: - log.warning( - f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)' - ) - failed.append(m) + log.warning(f'Setup failed for module "{m}"') + failed.append(m) + elif success or self.ignore_failed_deps: + log.debug( + f'Skipping dependency install for module "{m}" because it\'s already done (--force-deps to re-run)' + ) + succeeded.append(m) + else: + log.warning( + f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)' + ) + failed.append(m) finally: self.write_setup_status() @@ -127,25 +123,17 @@ async def install_module(self, module): success = True preloaded = self.all_modules_preloaded[module] - # ansible tasks - ansible_tasks = preloaded["deps"]["ansible"] - if ansible_tasks: + if ansible_tasks := preloaded["deps"]["ansible"]: success &= self.tasks(module, ansible_tasks) - # apt - deps_apt = preloaded["deps"]["apt"] - if deps_apt: + if deps_apt := preloaded["deps"]["apt"]: self.apt_install(deps_apt) - # shell - deps_shell = preloaded["deps"]["shell"] - if deps_shell: + if deps_shell := preloaded["deps"]["shell"]: success &= self.shell(module, deps_shell) - # pip - deps_pip = preloaded["deps"]["pip"] deps_pip_constraints = preloaded["deps"]["pip_constraints"] - if deps_pip: + if deps_pip := preloaded["deps"]["pip"]: success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints) return success @@ -211,7 +199,7 @@ def shell(self, module, commands): command["cmd"] += f" && touch {command_status_file}" tasks.append( { - "name": f"{module}.deps_shell step {i+1}", + "name": f"{module}.deps_shell step {i + 1}", "ansible.builtin.shell": command, "args": {"executable": "/bin/bash", "creates": str(command_status_file)}, } @@ -220,7 +208,7 @@ def shell(self, module, commands): if success: log.info(f"Successfully ran {len(commands):,} shell commands") else: - log.warning(f"Failed to run shell dependencies") + log.warning("Failed to run shell dependencies") return success def tasks(self, module, tasks): @@ -235,7 +223,7 @@ def tasks(self, module, tasks): def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): _ansible_args = {"ansible_connection": "local"} if ansible_args is not None: - _ansible_args.update(ansible_args) + _ansible_args |= ansible_args module_args = None if args: module_args = " ".join([f'{k}="{v}"' for k, v in args.items()]) @@ -243,19 +231,16 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): playbook = None if tasks: for task in tasks: - if "package" in task: - # special case for macos - if os_platform() == "darwin": - # don't sudo brew - task["become"] = False - # brew doesn't support update_cache - task["package"].pop("update_cache", "") + if "package" in task and os_platform() == "darwin": + task["become"] = False + # brew doesn't support update_cache + task["package"].pop("update_cache", "") playbook = {"hosts": "all", "tasks": tasks} log.debug(json.dumps(playbook, indent=2)) if self._sudo_password is not None: _ansible_args["ansible_become_password"] = self._sudo_password playbook_hash = self.parent_helper.sha1(str(playbook)).hexdigest() - data_dir = self.data_dir / (module if module else f"playbook_{playbook_hash}") + data_dir = self.data_dir / (module or f"playbook_{playbook_hash}") shutil.rmtree(data_dir, ignore_errors=True) self.parent_helper.mkdir(data_dir) @@ -276,17 +261,18 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): log.debug(f"Ansible status: {res.status}") log.debug(f"Ansible return code: {res.rc}") success = res.status == "successful" - err = "" - for e in res.events: - # if self.ansible_debug and not success: - # log.debug(json.dumps(e, indent=4)) - if e["event"] == "runner_on_failed": - err = e["event_data"]["res"]["msg"] - break + err = next( + ( + e["event_data"]["res"]["msg"] + for e in res.events + if e["event"] == "runner_on_failed" + ), + "", + ) return success, err def read_setup_status(self): - setup_status = dict() + setup_status = {} if self.setup_status_cache.is_file(): with open(self.setup_status_cache) as f: with suppress(Exception): @@ -315,16 +301,16 @@ def ensure_root(self, message=""): log.warning("Incorrect password") def install_core_deps(self): - to_install = set() self._install_sudo_askpass() # ensure tldextract data is cached self.parent_helper.tldextract("evilcorp.co.uk") # command: package_name core_deps = {"unzip": "unzip", "curl": "curl"} - for command, package_name in core_deps.items(): - if not self.parent_helper.which(command): - to_install.add(package_name) - if to_install: + if to_install := { + package_name + for command, package_name in core_deps.items() + if not self.parent_helper.which(command) + }: self.ensure_root() self.apt_install(list(to_install)) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index cf7252d8d..6a4726157 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -18,74 +18,75 @@ def __init__(self, baseline_url, parent_helper, method="GET", allow_redirects=Fa self._baselined = False async def _baseline(self): - if not self._baselined: - self._baselined = True - # vanilla URL - if self.include_cache_buster: - url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl() - else: - url_1 = self.baseline_url - baseline_1 = await self.parent_helper.request( - url_1, follow_redirects=self.allow_redirects, method=self.method - ) - await self.parent_helper.sleep(1) - # put random parameters in URL, headers, and cookies - get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)} - - if self.include_cache_buster: - get_params.update(self.gen_cache_buster()) - url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl() - baseline_2 = await self.parent_helper.request( - url_2, - headers={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, - cookies={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, - follow_redirects=self.allow_redirects, - method=self.method, + if self._baselined: + return + self._baselined = True + # vanilla URL + if self.include_cache_buster: + url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl() + else: + url_1 = self.baseline_url + baseline_1 = await self.parent_helper.request( + url_1, follow_redirects=self.allow_redirects, method=self.method + ) + await self.parent_helper.sleep(1) + # put random parameters in URL, headers, and cookies + get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)} + + if self.include_cache_buster: + get_params |= self.gen_cache_buster() + url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl() + baseline_2 = await self.parent_helper.request( + url_2, + headers={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, + cookies={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, + follow_redirects=self.allow_redirects, + method=self.method, + ) + + self.baseline = baseline_1 + + if baseline_1 is None or baseline_2 is None: + log.debug("HTTP error while establishing baseline, aborting") + raise HttpCompareError( + f"Can't get baseline from source URL: {url_1}:{baseline_1} / {url_2}:{baseline_2}" ) + if baseline_1.status_code != baseline_2.status_code: + log.debug("Status code not stable during baseline, aborting") + raise HttpCompareError("Can't get baseline from source URL") + try: + baseline_1_json = xmltodict.parse(baseline_1.text) + baseline_2_json = xmltodict.parse(baseline_2.text) + except ExpatError: + log.debug(f"Cant HTML parse for {self.baseline_url}. Switching to text parsing as a backup") + baseline_1_json = baseline_1.text.split("\n") + baseline_2_json = baseline_2.text.split("\n") - self.baseline = baseline_1 - - if baseline_1 is None or baseline_2 is None: - log.debug("HTTP error while establishing baseline, aborting") - raise HttpCompareError( - f"Can't get baseline from source URL: {url_1}:{baseline_1} / {url_2}:{baseline_2}" - ) - if baseline_1.status_code != baseline_2.status_code: - log.debug("Status code not stable during baseline, aborting") - raise HttpCompareError("Can't get baseline from source URL") - try: - baseline_1_json = xmltodict.parse(baseline_1.text) - baseline_2_json = xmltodict.parse(baseline_2.text) - except ExpatError: - log.debug(f"Cant HTML parse for {self.baseline_url}. Switching to text parsing as a backup") - baseline_1_json = baseline_1.text.split("\n") - baseline_2_json = baseline_2.text.split("\n") - - ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree") - self.ddiff_filters = [] - - for k, v in ddiff.items(): - for x in list(ddiff[k]): - log.debug(f"Added {k} filter for path: {x.path()}") - self.ddiff_filters.append(x.path()) - - self.baseline_json = baseline_1_json - - self.baseline_ignore_headers = [ - h.lower() - for h in [ - "date", - "last-modified", - "content-length", - "ETag", - "X-Pad", - "X-Backside-Transport", - ] + ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree") + self.ddiff_filters = [] + + for k, v in ddiff.items(): + for x in list(ddiff[k]): + log.debug(f"Added {k} filter for path: {x.path()}") + self.ddiff_filters.append(x.path()) + + self.baseline_json = baseline_1_json + + self.baseline_ignore_headers = [ + h.lower() + for h in [ + "date", + "last-modified", + "content-length", + "ETag", + "X-Pad", + "X-Backside-Transport", ] - dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers) + ] + dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers) - self.baseline_ignore_headers += [x.lower() for x in dynamic_headers] - self.baseline_body_distance = self.compare_body(baseline_1_json, baseline_2_json) + self.baseline_ignore_headers += [x.lower() for x in dynamic_headers] + self.baseline_body_distance = self.compare_body(baseline_1_json, baseline_2_json) def gen_cache_buster(self): return {self.parent_helper.rand_string(6): "1"} @@ -97,7 +98,7 @@ def compare_headers(self, headers_1, headers_2): for header, value in list(headers.items()): if header.lower() in self.baseline_ignore_headers: with suppress(KeyError): - log.debug(f'found ignored header "{header}" in headers_{i+1} and removed') + log.debug(f'found ignored header "{header}" in headers_{i + 1} and removed') del headers[header] ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree") @@ -119,12 +120,11 @@ def compare_body(self, content_1, content_2): if len(ddiff.keys()) == 0: return True - else: - log.debug(ddiff) - return False + log.debug(ddiff) + return False async def compare( - self, subject, headers=None, cookies=None, check_reflection=False, method="GET", allow_redirects=False + self, subject, headers=None, cookies=None, check_reflection=False, method="GET", allow_redirects=False ): """ Compares a URL with the baseline, with optional headers or cookies added @@ -180,13 +180,14 @@ async def compare( ) diff_reasons.append("code") - different_headers = self.compare_headers(self.baseline.headers, subject_response.headers) - if different_headers: + if different_headers := self.compare_headers( + self.baseline.headers, subject_response.headers + ): log.debug(f"headers were different, no match [{different_headers}]") diff_reasons.append("header") if self.compare_body(self.baseline_json, subject_json) == False: - log.debug(f"difference in HTML body, no match") + log.debug("difference in HTML body, no match") diff_reasons.append("body") @@ -202,7 +203,7 @@ async def canary_check(self, url, mode, rounds=6): await self._baseline() headers = None cookies = None - for i in range(0, rounds): + for _ in range(rounds): random_params = {self.parent_helper.rand_string(7): self.parent_helper.rand_string(6)} new_url = str(url) if mode == "getparam": diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 860853661..a028ad2a7 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -85,7 +85,7 @@ def __init__(self, parent_helper): try: self.resolver = BBOTAsyncResolver(_parent_helper=self.parent_helper) except Exception as e: - raise DNSError(f"Failed to create BBOT DNS resolver: {e}") + raise DNSError(f"Failed to create BBOT DNS resolver: {e}") from e self.timeout = self.parent_helper.config.get("dns_timeout", 5) self.retries = self.parent_helper.config.get("dns_retries", 1) self.abort_threshold = self.parent_helper.config.get("dns_abort_threshold", 50) @@ -95,10 +95,8 @@ def __init__(self, parent_helper): self._resolver_list = None # skip certain queries - dns_omit_queries = self.parent_helper.config.get("dns_omit_queries", None) - if not dns_omit_queries: - dns_omit_queries = [] - self.dns_omit_queries = dict() + dns_omit_queries = self.parent_helper.config.get("dns_omit_queries", None) or [] + self.dns_omit_queries = {} for d in dns_omit_queries: d = d.split(":") if len(d) == 2: @@ -110,12 +108,10 @@ def __init__(self, parent_helper): except KeyError: self.dns_omit_queries[rdtype] = {query} - self.wildcard_ignore = self.parent_helper.config.get("dns_wildcard_ignore", None) - if not self.wildcard_ignore: - self.wildcard_ignore = [] + self.wildcard_ignore = self.parent_helper.config.get("dns_wildcard_ignore", None) or [] self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) self.wildcard_tests = self.parent_helper.config.get("dns_wildcard_tests", 5) - self._wildcard_cache = dict() + self._wildcard_cache = {} # since wildcard detection takes some time, This is to prevent multiple # modules from kicking off wildcard detection for the same domain at the same time self._wildcard_lock = NamedLock() @@ -124,10 +120,10 @@ def __init__(self, parent_helper): self._last_connectivity_warning = time.time() # keeps track of warnings issued for wildcard detection to prevent duplicate warnings self._dns_warnings = set() - self._errors = dict() + self._errors = {} self.fallback_nameservers_file = self.parent_helper.wordlist_dir / "nameservers.txt" self._debug = self.parent_helper.config.get("dns_debug", False) - self._dummy_modules = dict() + self._dummy_modules = {} self._dns_cache = self.parent_helper.CacheDict(max_size=100000) self._event_cache = self.parent_helper.CacheDict(max_size=10000) self._event_cache_locks = NamedLock() @@ -225,7 +221,7 @@ async def resolve_raw(self, query, **kwargs): types = self.all_rdtypes else: types = [t.strip().upper()] - elif any([isinstance(t, x) for x in (list, tuple)]): + elif any(isinstance(t, x) for x in (list, tuple)): types = [str(_).strip().upper() for _ in t] for t in types: r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) @@ -270,10 +266,9 @@ async def _resolve_hostname(self, query, **kwargs): rdtype = kwargs.get("rdtype", "A") # skip certain queries if requested - if rdtype in self.dns_omit_queries: - if any(h == query or query.endswith(f".{h}") for h in self.dns_omit_queries[rdtype]): - self.debug(f"Skipping {rdtype}:{query} because it's omitted in the config") - return results, errors + if rdtype in self.dns_omit_queries and any(h == query or query.endswith(f".{h}") for h in self.dns_omit_queries[rdtype]): + self.debug(f"Skipping {rdtype}:{query} because it's omitted in the config") + return results, errors parent = self.parent_helper.parent_domain(query) retries = kwargs.pop("retries", self.retries) @@ -306,10 +301,10 @@ async def _resolve_hostname(self, query, **kwargs): self._errors[parent_hash] = 0 break except ( - dns.resolver.NoNameservers, - dns.exception.Timeout, - dns.resolver.LifetimeTimeout, - TimeoutError, + dns.resolver.NoNameservers, + dns.exception.Timeout, + dns.resolver.LifetimeTimeout, + TimeoutError, ) as e: try: self._errors[parent_hash] += 1 @@ -374,10 +369,10 @@ async def _resolve_ip(self, query, **kwargs): self._dns_cache[dns_cache_hash] = results break except ( - dns.exception.Timeout, - dns.resolver.LifetimeTimeout, - dns.resolver.NoNameservers, - TimeoutError, + dns.exception.Timeout, + dns.resolver.LifetimeTimeout, + dns.resolver.NoNameservers, + TimeoutError, ) as e: errors.append(e) # don't retry if we get a SERVFAIL @@ -433,7 +428,7 @@ async def handle_wildcard_event(self, event, children): if not is_ip(event.host) and children: if wildcard_rdtypes: # these are the rdtypes that successfully resolve - resolved_rdtypes = set([c.upper() for c in children]) + resolved_rdtypes = {c.upper() for c in children} # these are the rdtypes that have wildcards wildcard_rdtypes_set = set(wildcard_rdtypes) # consider the event a full wildcard if all its records are wildcards @@ -441,19 +436,26 @@ async def handle_wildcard_event(self, event, children): if resolved_rdtypes: event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) - if event_is_wildcard: - if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = self.parent_helper.parent_domain(event_host) - for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): - if _is_wildcard: - wildcard_parent = _parent_domain - break - wildcard_data = f"_wildcard.{wildcard_parent}" - if wildcard_data != event.data: - log.debug( - f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"' - ) - event.data = wildcard_data + if event_is_wildcard and (event.type in ( + "DNS_NAME", + ) and "_wildcard" not in event.data.split(".")): + wildcard_parent = next( + ( + _parent_domain + for rdtype, ( + _is_wildcard, + _parent_domain, + ) in wildcard_rdtypes.items() + if _is_wildcard + ), + self.parent_helper.parent_domain(event_host), + ) + wildcard_data = f"_wildcard.{wildcard_parent}" + if wildcard_data != event.data: + log.debug( + f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"' + ) + event.data = wildcard_data else: # check if this domain is using wildcard dns event_target = "target" in event.tags @@ -517,11 +519,10 @@ async def resolve_event(self, event, minimal=False): if self.parent_helper.is_ip(event.host): if not minimal: types = ("PTR",) + elif event.type == "DNS_NAME" and not minimal: + types = self.all_rdtypes else: - if event.type == "DNS_NAME" and not minimal: - types = self.all_rdtypes - else: - types = ("A", "AAAA") + types = ("A", "AAAA") if types: tasks = [self.resolve_raw(event_host, type=t, use_cache=True) for t in types] @@ -542,11 +543,10 @@ async def resolve_event(self, event, minimal=False): if t: ip = self.parent_helper.make_ip_type(t) - if rdtype in ("A", "AAAA", "CNAME"): + if rdtype in {"A", "AAAA", "CNAME"}: with contextlib.suppress(ValidationError): - if self.parent_helper.is_ip(ip): - if self.parent_helper.scan.whitelisted(ip): - event_whitelisted = True + if self.parent_helper.is_ip(ip) and self.parent_helper.scan.whitelisted(ip): + event_whitelisted = True with contextlib.suppress(ValidationError): if self.parent_helper.scan.blacklisted(ip): event_blacklisted = True @@ -578,7 +578,7 @@ async def resolve_event(self, event, minimal=False): if not is_ip(event_host) and "resolved" not in event_tags: event_tags.add("unresolved") # check for private IPs - for rdtype, ips in dns_children.items(): + for ips in dns_children.values(): for ip in ips: try: ip = ipaddress.ip_address(ip) @@ -659,7 +659,7 @@ async def resolve_batch(self, queries, **kwargs): queries = list(queries) batch_size = 250 for i in range(0, len(queries), batch_size): - batch = queries[i : i + batch_size] + batch = queries[i: i + batch_size] tasks = [self._resolve_batch_coro_wrapper(q, **kwargs) for q in batch] async for task in as_completed(tasks): yield await task @@ -691,7 +691,7 @@ def extract_targets(self, record): """ results = set() rdtype = str(record.rdtype.name).upper() - if rdtype in ("A", "AAAA", "NS", "CNAME", "PTR"): + if rdtype in {"A", "AAAA", "NS", "CNAME", "PTR"}: results.add((rdtype, self._clean_dns_record(record))) elif rdtype == "SOA": results.add((rdtype, self._clean_dns_record(record.mname))) @@ -820,7 +820,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): query = self._clean_dns_record(query) # skip check if it's an IP - if is_ip(query) or not "." in query: + if is_ip(query) or "." not in query: return {} # skip check if the query is a domain if is_domain(query): @@ -831,7 +831,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): rdtypes_to_check = [rdtype] if rdtype is not None else self.all_rdtypes - base_query_ips = dict() + base_query_ips = {} # if the caller hasn't already done the work of resolving the IPs if ips is None: # then resolve the query for all rdtypes @@ -851,12 +851,10 @@ async def is_wildcard(self, query, ips=None, rdtype=None): base_query_results.add(t) if base_query_results: base_query_ips[__rdtype] = base_query_results - else: - # otherwise, we can skip all that - cleaned_ips = set([self._clean_dns_record(ip) for ip in ips]) - if not cleaned_ips: - raise ValueError("Valid IPs must be specified") + elif cleaned_ips := {self._clean_dns_record(ip) for ip in ips}: base_query_ips[rdtype] = cleaned_ips + else: + raise ValueError("Valid IPs must be specified") if not base_query_ips: return result @@ -864,7 +862,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): # we can compare the IPs to the ones we have on file for wildcards # for every parent domain, starting with the shortest - try: + with suppress(DNSWildcardBreak): for host in parents[::-1]: # for every rdtype for _rdtype in list(base_query_ips): @@ -884,20 +882,21 @@ async def is_wildcard(self, query, ips=None, rdtype=None): # if our IPs match the wildcard ones, then ladies and gentlemen we have a wildcard is_wildcard = any(r in wildcard_ips for r in query_ips) - if is_wildcard and not result.get(_rdtype, (None, None))[0] is True: + if ( + is_wildcard + and result.get(_rdtype, (None, None))[0] + is not True + ): result[_rdtype] = (True, host) # if we've reached a point where the dns name is a complete wildcard, class can be dismissed early base_query_rdtypes = set(base_query_ips) - wildcard_rdtypes_set = set([k for k, v in result.items() if v[0] is True]) + wildcard_rdtypes_set = {k for k, v in result.items() if v[0] is True} if base_query_rdtypes and wildcard_rdtypes_set and base_query_rdtypes == wildcard_rdtypes_set: log.debug( f"Breaking from wildcard detection for {query} at {host} because base query rdtypes ({base_query_rdtypes}) == wildcard rdtypes ({wildcard_rdtypes_set})" ) raise DNSWildcardBreak() - except DNSWildcardBreak: - pass - return result async def is_wildcard_domain(self, domain, log_info=False): @@ -938,7 +937,7 @@ async def is_wildcard_domain(self, domain, log_info=False): # make a list of its parents parents = list(domain_parents(domain, include_self=True)) # and check each of them, beginning with the highest parent (i.e. the root domain) - for i, host in enumerate(parents[::-1]): + for host in parents[::-1]: # have we checked this host before? host_hash = hash(host) async with self._wildcard_lock.lock(host_hash): @@ -966,7 +965,7 @@ async def is_wildcard_domain(self, domain, log_info=False): results = await task if results: is_wildcard = True - if not rdtype in wildcard_results: + if rdtype not in wildcard_results: wildcard_results[rdtype] = set() wildcard_results[rdtype].update(results) # we know this rdtype is a wildcard @@ -975,7 +974,7 @@ async def is_wildcard_domain(self, domain, log_info=False): rdtypes_to_check.remove(rdtype) self._wildcard_cache.update({host_hash: wildcard_results}) - wildcard_domain_results.update({host: wildcard_results}) + wildcard_domain_results[host] = wildcard_results if is_wildcard: wildcard_rdtypes_str = ",".join(sorted([t.upper() for t, r in wildcard_results.items() if r])) log_fn = log.verbose @@ -1000,9 +999,8 @@ async def _connectivity_check(self, interval=5): >>> await _connectivity_check() True """ - if self._last_dns_success is not None: - if time.time() - self._last_dns_success < interval: - return True + if self._last_dns_success is not None and time.time() - self._last_dns_success < interval: + return True dns_server_working = [] async with self._dns_connectivity_lock: with suppress(Exception): @@ -1011,7 +1009,7 @@ async def _connectivity_check(self, interval=5): self._last_dns_success = time.time() return True if time.time() - self._last_connectivity_warning > interval: - log.warning(f"DNS queries are failing, please check your internet connection") + log.warning("DNS queries are failing, please check your internet connection") self._last_connectivity_warning = time.time() self._errors.clear() return False diff --git a/bbot/core/helpers/files.py b/bbot/core/helpers/files.py index 438f74112..5927d2fab 100644 --- a/bbot/core/helpers/files.py +++ b/bbot/core/helpers/files.py @@ -6,7 +6,6 @@ from .misc import rm_at_exit - log = logging.getLogger("bbot.core.helpers.files") @@ -84,7 +83,7 @@ def _feed_pipe(self, pipe, content, text=True): for c in content: p.write(decode_fn(c) + newline) except BrokenPipeError: - log.debug(f"Broken pipe in _feed_pipe()") + log.debug("Broken pipe in _feed_pipe()") except ValueError: log.debug(f"Error _feed_pipe(): {traceback.format_exc()}") except KeyboardInterrupt: diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index eef59e5d3..83031439a 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -13,7 +13,6 @@ from ...modules.base import BaseModule from .depsinstaller import DepsInstaller - log = logging.getLogger("bbot.core.helpers") @@ -156,9 +155,9 @@ def __getattribute__(self, attr): try: # then try web return getattr(self.web, attr) - except AttributeError: + except AttributeError as e: # then die - raise AttributeError(f'Helper has no attribute "{attr}"') + raise AttributeError(f'Helper has no attribute "{attr}"') from e class DummyModule(BaseModule): diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index 695fd6260..2cf701257 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -153,7 +153,7 @@ async def register(self, callback=None): break if not self.server: - raise InteractshError(f"Failed to register with an interactsh server") + raise InteractshError("Failed to register with an interactsh server") log.info( f"Successfully registered to interactsh server {self.server} with correlation_id {self.correlation_id} [{self.domain}]" @@ -179,7 +179,7 @@ async def deregister(self): >>> await interactsh_client.deregister() """ if not self.server or not self.correlation_id or not self.secret: - raise InteractshError(f"Missing required information to deregister") + raise InteractshError("Missing required information to deregister") headers = {} if self.token: @@ -224,7 +224,7 @@ async def poll(self): ] """ if not self.server or not self.correlation_id or not self.secret: - raise InteractshError(f"Missing required information to poll") + raise InteractshError("Missing required information to poll") headers = {} if self.token: @@ -235,8 +235,7 @@ async def poll(self): ) ret = [] - data_list = r.json().get("data", None) - if data_list: + if data_list := r.json().get("data", None): aes_key = r.json()["aes_key"] for data in data_list: diff --git a/bbot/core/helpers/logger.py b/bbot/core/helpers/logger.py index b70d4b4b4..0874670eb 100644 --- a/bbot/core/helpers/logger.py +++ b/bbot/core/helpers/logger.py @@ -33,16 +33,15 @@ def colorize(s, level="INFO"): seq = color_mapping.get(level, 15) # default white - colored = f"{color_prefix}{seq}m{s}{color_suffix}" - return colored + return f"{color_prefix}{seq}m{s}{color_suffix}" def log_to_stderr(msg, level="INFO", logname=True): """ Print to stderr with BBOT logger colors """ - levelname = level.upper() - if not any(x in sys.argv for x in ("-s", "--silent")): + if all(x not in sys.argv for x in ("-s", "--silent")): + levelname = level.upper() levelshort = f"[{loglevel_mapping.get(level, 'INFO')}]" levelshort = f"{colorize(levelshort, level=levelname)}" if levelname == "CRITICAL" or levelname.startswith("HUGE"): diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index ecbceaa33..89e3b8ce5 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1,43 +1,43 @@ -import os -import re -import sys -import copy -import idna -import json +import asyncio import atexit import codecs -import psutil +import contextlib +import copy +import difflib +import inspect +import ipaddress +import json +import os +import platform import random +import re import shutil import signal import string -import asyncio -import difflib -import inspect -import logging -import platform -import ipaddress -import traceback import subprocess as sp -from pathlib import Path -from itertools import islice -from datetime import datetime -from tabulate import tabulate -import wordninja as _wordninja -from contextlib import suppress -import cloudcheck as _cloudcheck -import tldextract as _tldextract +import sys +import traceback import xml.etree.ElementTree as ET +from asyncio import create_task, gather, sleep, wait_for # noqa from collections.abc import Mapping +from datetime import datetime from hashlib import sha1 as hashlib_sha1 -from asyncio import create_task, gather, sleep, wait_for # noqa +from itertools import islice +from pathlib import Path from urllib.parse import urlparse, quote, unquote, urlunparse # noqa F401 -from .url import * # noqa F401 -from .. import errors -from .logger import log_to_stderr +import cloudcheck as _cloudcheck +import idna +import psutil +import tldextract as _tldextract +import wordninja as _wordninja +from tabulate import tabulate + from . import regexes as bbot_regexes +from .logger import log_to_stderr from .names_generator import random_name, names, adjectives # noqa F401 +from .url import * # noqa F401 +from .. import errors log = logging.getLogger("bbot.core.helpers.misc") @@ -67,9 +67,7 @@ def is_domain(d): """ d, _ = split_host_port(d) extracted = tldextract(d) - if extracted.domain and not extracted.subdomain: - return True - return False + return bool(extracted.domain and not extracted.subdomain) def is_subdomain(d): @@ -97,9 +95,7 @@ def is_subdomain(d): """ d, _ = split_host_port(d) extracted = tldextract(d) - if extracted.domain and extracted.subdomain: - return True - return False + return bool(extracted.domain and extracted.subdomain) def is_ptr(d): @@ -146,10 +142,7 @@ def is_url(u): False """ u = str(u) - for r in bbot_regexes.event_type_regexes["URL"]: - if r.match(u): - return True - return False + return any(r.match(u) for r in bbot_regexes.event_type_regexes["URL"]) uri_regex = re.compile(r"^([a-z0-9]{2,20})://", re.I) @@ -184,9 +177,7 @@ def is_uri(u, return_scheme=False): """ match = uri_regex.match(u) if return_scheme: - if match: - return match.groups()[0].lower() - return "" + return match.groups()[0].lower() if match else "" return bool(match) @@ -382,7 +373,7 @@ def url_parents(u): parent_list = [] while 1: parent = parent_url(u) - if parent == None: + if parent is None: return parent_list elif parent not in parent_list: parent_list.append(parent) @@ -457,7 +448,9 @@ def domain_stem(domain): - Utilizes the `tldextract` function for domain parsing. """ parsed = tldextract(str(domain)) - return f".".join(parsed.subdomain.split(".") + parsed.domain.split(".")).strip(".") + return ".".join( + parsed.subdomain.split(".") + parsed.domain.split(".") + ).strip(".") def ip_network_parents(i, include_self=False): @@ -527,9 +520,7 @@ def is_dns_name(d): d = smart_decode(d) if bbot_regexes.hostname_regex.match(d): return True - if bbot_regexes.dns_name_regex.match(d): - return True - return False + return bool(bbot_regexes.dns_name_regex.match(d)) def is_ip(d, version=None): @@ -553,15 +544,12 @@ def is_ip(d, version=None): >>> is_ip('evilcorp.com') False """ - if isinstance(d, (ipaddress.IPv4Address, ipaddress.IPv6Address)): - if version is None or version == d.version: - return True - try: + if isinstance(d, (ipaddress.IPv4Address, ipaddress.IPv6Address)) and (version is None or version == d.version): + return True + with contextlib.suppress(Exception): ip = ipaddress.ip_address(d) if version is None or ip.version == version: return True - except Exception: - pass return False @@ -583,7 +571,7 @@ def is_ip_type(i): >>> is_ip_type("192.168.1.0/24") False """ - return isinstance(i, ipaddress._BaseV4) or isinstance(i, ipaddress._BaseV6) + return isinstance(i, (ipaddress._BaseV4, ipaddress._BaseV6)) def make_ip_type(s): @@ -666,13 +654,12 @@ def host_in_host(host1, host2): host2_ip_type = is_ip_type(host2) # if both hosts are IP types if host1_ip_type and host2_ip_type: - if not host1.version == host2.version: + if host1.version != host2.version: return False host1_net = ipaddress.ip_network(host1) host2_net = ipaddress.ip_network(host2) return host1_net.subnet_of(host2_net) - # else hostnames elif not (host1_ip_type or host2_ip_type): host2_len = len(host2.split(".")) host1_truncated = ".".join(host1.split(".")[-host2_len:]) @@ -851,10 +838,7 @@ def extract_params_json(json_data): if isinstance(value, (dict, list)): stack.append(value) elif isinstance(current_data, list): - for item in current_data: - if isinstance(item, (dict, list)): - stack.append(item) - + stack.extend(item for item in current_data if isinstance(item, (dict, list))) return keys @@ -887,8 +871,7 @@ def extract_params_xml(xml_data): while stack: current_element = stack.pop() tags.add(current_element.tag) - for child in current_element: - stack.append(child) + stack.extend(iter(current_element)) return tags @@ -931,9 +914,7 @@ def extract_params_html(html_data): log.debug(f"FOUND PARAM ({i}) IN JQUERY GET PARAMS") yield i - # check for jquery post parameters - jquery_post = bbot_regexes.jquery_post_regex.findall(html_data) - if jquery_post: + if jquery_post := bbot_regexes.jquery_post_regex.findall(html_data): for i in jquery_post: for x in i.split(","): s = x.split(":")[0].rstrip() @@ -995,9 +976,8 @@ def extract_words(data, acronyms=True, wordninja=True, model=None, max_length=10 # subword_slice = "".join(subwords[s:e]) # words.add(subword_slice) # blacklanternsecurity --> bls - if acronyms: - if len(subwords) > 1: - words.add("".join([c[0] for c in subwords if len(c) > 0])) + if acronyms and len(subwords) > 1: + words.add("".join([c[0] for c in subwords if len(c) > 0])) return words @@ -1026,9 +1006,7 @@ def closest_match(s, choices, n=1, cutoff=0.0): matches = difflib.get_close_matches(s, choices, n=n, cutoff=cutoff) if not choices or not matches: return - if n == 1: - return matches[0] - return matches + return matches[0] if n == 1 else matches def match_and_exit(s, choices, msg=None, loglevel="HUGEWARNING", exitcode=2): @@ -1126,7 +1104,7 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True): >>> chain_lists(["a,file.txt", "c,d"], try_files=True) ['a', 'f_line1', 'f_line2', 'f_line3', 'c', 'd'] """ - final_list = dict() + final_list = {} for entry in l: for s in entry.split(","): f = s.strip() @@ -1274,9 +1252,7 @@ def make_netloc(host, port): """ if is_ip(host, version=6): host = f"[{host}]" - if port is None: - return host - return f"{host}:{port}" + return host if port is None else f"{host}:{port}" def which(*executables): @@ -1293,8 +1269,7 @@ def which(*executables): "/usr/bin/python" """ for e in executables: - location = shutil.which(e) - if location: + if location := shutil.which(e): return location @@ -1373,8 +1348,8 @@ def search_dict_values(d, *regexes): ["https://www.evilcorp.com"] """ - results = set() if isinstance(d, str): + results = set() for r in regexes: for match in r.finditer(d): result = match.group() @@ -1418,11 +1393,16 @@ def filter_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): if isinstance(d, dict): for key in d: if key in key_names or (fuzzy and any(k in key for k in key_names)): - if not any(k in exclude_keys for k in [key, _prev_key]): + if all(k not in exclude_keys for k in [key, _prev_key]): ret[key] = copy.deepcopy(d[key]) - elif isinstance(d[key], list) or isinstance(d[key], dict): - child = filter_dict(d[key], *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys) - if child: + elif isinstance(d[key], (list, dict)): + if child := filter_dict( + d[key], + *key_names, + fuzzy=fuzzy, + _prev_key=key, + exclude_keys=exclude_keys, + ): ret[key] = child return ret @@ -1494,7 +1474,7 @@ def split_list(alist, wanted_parts=2): [[1, 2], [3, 4, 5]] """ length = len(alist) - return [alist[i * length // wanted_parts : (i + 1) * length // wanted_parts] for i in range(wanted_parts)] + return [alist[i * length // wanted_parts: (i + 1) * length // wanted_parts] for i in range(wanted_parts)] def mkdir(path, check_writable=True, raise_error=True): @@ -1526,7 +1506,9 @@ def mkdir(path, check_writable=True, raise_error=True): return True except Exception as e: if raise_error: - raise errors.DirectoryCreationError(f"Failed to create directory at {path}: {e}") + raise errors.DirectoryCreationError( + f"Failed to create directory at {path}: {e}" + ) from e finally: with suppress(Exception): touchfile.unlink() @@ -1583,10 +1565,7 @@ def get_file_extension(s): """ s = str(s).lower().strip() rightmost_section = s.rsplit("/", 1)[-1] - if "." in rightmost_section: - extension = rightmost_section.rsplit(".", 1)[-1] - return extension - return "" + return rightmost_section.rsplit(".", 1)[-1] if "." in rightmost_section else "" def backup_file(filename, max_backups=10): @@ -1643,12 +1622,8 @@ def latest_mtime(d): """ d = Path(d).resolve() mtimes = [d.lstat().st_mtime] - if d.is_dir(): - to_list = d.glob("**/*") - else: - to_list = [d] - for e in to_list: - mtimes.append(e.lstat().st_mtime) + to_list = d.glob("**/*") if d.is_dir() else [d] + mtimes.extend(e.lstat().st_mtime for e in to_list) return max(mtimes) @@ -1669,9 +1644,7 @@ def filesize(f): 1024 """ f = Path(f) - if f.is_file(): - return f.stat().st_size - return 0 + return f.stat().st_size if f.is_file() else 0 def clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, raise_error=False): @@ -1705,7 +1678,7 @@ def clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, except Exception as e: msg = f"Failed to delete directory: {path}, {e}" if raise_error: - raise errors.DirectoryDeletionError() + raise errors.DirectoryDeletionError() from e log.warning(msg) @@ -1763,12 +1736,10 @@ def extract_host(s): ) """ s = smart_decode(s) - match = bbot_regexes.extract_host_regex.search(s) - - if match: + if match := bbot_regexes.extract_host_regex.search(s): hostname = match.group(1) before = s[: match.start(1)] - after = s[match.end(1) :] + after = s[match.end(1):] host, port = split_host_port(hostname) netloc = make_netloc(host, port) if netloc != hostname: @@ -1794,11 +1765,8 @@ def smart_encode_punycode(text: str) -> str: if host is None: return text - try: + with contextlib.suppress(UnicodeError): host = idna.encode(host).decode(errors="ignore") - except UnicodeError: - pass # If encoding fails, leave the host as it is - return f"{before}{host}{after}" @@ -1810,11 +1778,8 @@ def smart_decode_punycode(text: str) -> str: if host is None: return text - try: + with contextlib.suppress(UnicodeError): host = idna.decode(host) - except UnicodeError: - pass # If decoding fails, leave the host as it is - return f"{before}{host}{after}" @@ -1903,9 +1868,9 @@ def make_table(*args, **kwargs): tablefmt = os.environ.get("BBOT_TABLE_FORMAT", None) defaults = {"tablefmt": "grid", "disable_numparse": True, "maxcolwidths": None} if tablefmt is None: - defaults.update({"maxcolwidths": 40}) + defaults["maxcolwidths"] = 40 else: - defaults.update({"tablefmt": tablefmt}) + defaults["tablefmt"] = tablefmt for k, v in defaults.items(): if k not in kwargs: kwargs[k] = v @@ -1945,9 +1910,7 @@ def human_timedelta(d): result.append(f"{minutes:,} minute" + ("s" if minutes > 1 else "")) if seconds: result.append(f"{seconds:,} second" + ("s" if seconds > 1 else "")) - ret = ", ".join(result) - if not ret: - ret = "0 seconds" + ret = ", ".join(result) or "0 seconds" return ret @@ -1969,15 +1932,10 @@ def bytes_to_human(_bytes): '1.15GB' """ sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"] - units = {} - for count, size in enumerate(sizes): - units[size] = pow(1024, count) + units = {size: pow(1024, count) for count, size in enumerate(sizes)} for size in sizes: if abs(_bytes) < 1024.0: - if size == sizes[0]: - _bytes = str(int(_bytes)) - else: - _bytes = f"{_bytes:.2f}" + _bytes = str(int(_bytes)) if size == sizes[0] else f"{_bytes:.2f}" return f"{_bytes}{size}" _bytes /= 1024 raise ValueError(f'Unable to convert "{_bytes}" to human filesize') @@ -2015,14 +1973,12 @@ def human_to_bytes(filesize): if len(size) == 2: units[size[0]] = size_increment match = filesize_regex.match(filesize) - try: + with contextlib.suppress(KeyError): if match: num, size = match.groups() size = size.upper() size_increment = units[size] return int(float(num) * size_increment) - except KeyError: - pass raise ValueError(f'Unable to convert filesize "{filesize}" to bytes') @@ -2078,9 +2034,7 @@ def os_platform_friendly(): 'macOS' """ p = os_platform() - if p == "darwin": - return "macOS" - return p + return "macOS" if p == "darwin" else p tag_filter_regex = re.compile(r"[^a-z0-9]+") diff --git a/bbot/core/helpers/modules.py b/bbot/core/helpers/modules.py index c6cc52f42..9d5993ca5 100644 --- a/bbot/core/helpers/modules.py +++ b/bbot/core/helpers/modules.py @@ -55,14 +55,14 @@ def preload(self, module_dir): module_dir = Path(module_dir) for module_file in list_files(module_dir, filter=self.file_filter): if module_dir.name == "modules": - namespace = f"bbot.modules" + namespace = "bbot.modules" else: namespace = f"bbot.modules.{module_dir.name}" try: preloaded = self.preload_module(module_file) module_type = "scan" - if module_dir.name in ("output", "internal"): - module_type = str(module_dir.name) + if module_dir.name in {"output", "internal"}: + module_type = module_dir.name elif module_dir.name not in ("modules"): preloaded["flags"] = list(set(preloaded["flags"] + [module_dir.name])) preloaded["type"] = module_type @@ -79,11 +79,11 @@ def preload(self, module_dir): def preloaded(self, type=None): preloaded = {} - if type is not None: - preloaded = {k: v for k, v in self._preloaded.items() if self.check_type(k, type)} - else: - preloaded = dict(self._preloaded) - return preloaded + return ( + {k: v for k, v in self._preloaded.items() if self.check_type(k, type)} + if type is not None + else dict(self._preloaded) + ) def configs(self, type=None): configs = {} @@ -169,58 +169,90 @@ def preload_module(self, module_file): # class attributes that are dictionaries if type(class_attr) == ast.Assign and type(class_attr.value) == ast.Dict: # module options - if any([target.id == "options" for target in class_attr.targets]): - config.update(ast.literal_eval(class_attr.value)) + if any( + target.id == "options" for target in class_attr.targets + ): + config |= ast.literal_eval(class_attr.value) # module options - if any([target.id == "options_desc" for target in class_attr.targets]): - options_desc.update(ast.literal_eval(class_attr.value)) + if any( + target.id == "options_desc" + for target in class_attr.targets + ): + options_desc |= ast.literal_eval(class_attr.value) # module metadata - if any([target.id == "meta" for target in class_attr.targets]): + if any( + target.id == "meta" for target in class_attr.targets + ): meta = ast.literal_eval(class_attr.value) # class attributes that are lists if type(class_attr) == ast.Assign and type(class_attr.value) == ast.List: # flags - if any([target.id == "flags" for target in class_attr.targets]): - for flag in class_attr.value.elts: - if type(flag.value) == str: - flags.append(flag.value) + if any( + target.id == "flags" for target in class_attr.targets + ): + flags.extend( + flag.value + for flag in class_attr.value.elts + if type(flag.value) == str + ) # watched events - if any([target.id == "watched_events" for target in class_attr.targets]): - for event_type in class_attr.value.elts: - if type(event_type.value) == str: - watched_events.append(event_type.value) + if any( + target.id == "watched_events" + for target in class_attr.targets + ): + watched_events.extend( + event_type.value + for event_type in class_attr.value.elts + if type(event_type.value) == str + ) # produced events - if any([target.id == "produced_events" for target in class_attr.targets]): - for event_type in class_attr.value.elts: - if type(event_type.value) == str: - produced_events.append(event_type.value) + if any( + target.id == "produced_events" + for target in class_attr.targets + ): + produced_events.extend( + event_type.value + for event_type in class_attr.value.elts + if type(event_type.value) == str + ) # python dependencies - if any([target.id == "deps_pip" for target in class_attr.targets]): + if any( + target.id == "deps_pip" + for target in class_attr.targets + ): for python_dep in class_attr.value.elts: if type(python_dep.value) == str: pip_deps.append(python_dep.value) - if any([target.id == "deps_pip_constraints" for target in class_attr.targets]): + if any( + target.id == "deps_pip_constraints" + for target in class_attr.targets + ): for python_dep in class_attr.value.elts: if type(python_dep.value) == str: pip_deps_constraints.append(python_dep.value) - # apt dependencies - elif any([target.id == "deps_apt" for target in class_attr.targets]): + elif any( + target.id == "deps_apt" + for target in class_attr.targets + ): for apt_dep in class_attr.value.elts: if type(apt_dep.value) == str: apt_deps.append(apt_dep.value) - # bash dependencies - elif any([target.id == "deps_shell" for target in class_attr.targets]): + elif any( + target.id == "deps_shell" + for target in class_attr.targets + ): for shell_dep in class_attr.value.elts: shell_deps.append(ast.literal_eval(shell_dep)) - # ansible playbook - elif any([target.id == "deps_ansible" for target in class_attr.targets]): + elif any( + target.id == "deps_ansible" + for target in class_attr.targets + ): ansible_tasks = ast.literal_eval(class_attr.value) for task in ansible_tasks: - if not "become" in task: + if "become" not in task: task["become"] = False - # don't sudo brew elif os_platform() == "darwin" and ("package" in task and task.get("become", False) == True): task["become"] = False preloaded_data = { @@ -241,7 +273,7 @@ def preload_module(self, module_file): "sudo": len(apt_deps) > 0, } if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any( - x == True for x in search_dict_by_key("ansible_become", ansible_tasks) + x == True for x in search_dict_by_key("ansible_become", ansible_tasks) ): preloaded_data["sudo"] = True return preloaded_data @@ -282,14 +314,12 @@ def load_module(self, module_name): with suppress(AttributeError): # if it has watched_events and produced_events if all( - type(a) == list - for a in (getattr(value, "watched_events", None), getattr(value, "produced_events", None)) - ): - # and if its variable name matches its filename - if value.__name__.lower() == module_name.lower(): - value._name = module_name - # then we have a module - return value + type(a) == list + for a in (getattr(value, "watched_events", None), getattr(value, "produced_events", None)) + ) and value.__name__.lower() == module_name.lower(): + value._name = module_name + # then we have a module + return value def recommend_dependencies(self, modules): """ @@ -303,8 +333,7 @@ def recommend_dependencies(self, modules): watched = {} produced = {} for modname in modules: - preloaded = self._preloaded.get(modname) - if preloaded: + if preloaded := self._preloaded.get(modname): for event_type in preloaded.get("watched_events", []): self.add_or_create(watched, event_type, modname) for event_type in preloaded.get("produced_events", []): @@ -346,11 +375,7 @@ def recommend_dependencies(self, modules): return resolve_choices def check_dependency(self, event_type, modname, produced): - if event_type not in produced: - return False - if produced[event_type] == {modname}: - return False - return True + return produced[event_type] != {modname} if event_type in produced else False @staticmethod def add_or_create(d, k, *items): @@ -444,8 +469,7 @@ def flags(self, flags=None): _flags[flag] = {module_name} _flags = sorted(_flags.items(), key=lambda x: x[0]) - _flags = sorted(_flags, key=lambda x: len(x[-1]), reverse=True) - return _flags + return sorted(_flags, key=lambda x: len(x[-1]), reverse=True) def flags_table(self, flags=None): table = [] diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 49ed866d6..e00ab5acb 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -665,8 +665,8 @@ def random_name(): name = random.choice(names) adjective = random.choice(adjectives) - if adjective == "unchained": - scan_name = f"{name}_{adjective}" - else: - scan_name = f"{adjective}_{name}" - return scan_name + return ( + f"{name}_{adjective}" + if adjective == "unchained" + else f"{adjective}_{name}" + ) diff --git a/bbot/core/helpers/ntlm.py b/bbot/core/helpers/ntlm.py index e4d9cd1ca..4b100cde4 100644 --- a/bbot/core/helpers/ntlm.py +++ b/bbot/core/helpers/ntlm.py @@ -16,7 +16,7 @@ def __init__(self, pos_tup, raw): self.length = length self.alloc = alloc self.offset = offset - self.raw = raw[offset : offset + length] + self.raw = raw[offset: offset + length] self.utf16 = False if len(self.raw) >= 2 and self.raw[1] == "\0": @@ -42,41 +42,48 @@ def decode_ntlm_challenge(st): nxt = st[40:48] if len(nxt) == 8: - hdr_tup = struct.unpack("= self.log_interval: - log.verbose(f"{self.name} rate limit threshold ({self.rate*10:.1f}/s) reached") + log.verbose(f"{self.name} rate limit threshold ({self.rate * 10:.1f}/s) reached") self.last_notification = now # Rate limit for the current 0.1 second interval has been reached, wait until the next interval await asyncio.sleep(self.current_timestamp + 0.1 - time.time()) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 313626094..733420852 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -62,33 +62,32 @@ ( (k, tuple(re.compile(r, re.I) for r in regexes)) for k, regexes in ( + ( + "DNS_NAME", ( - "DNS_NAME", - ( - r"^" + _dns_name_regex + r"$", - r"^" + _hostname_regex + r"$", - ), + r"^" + _dns_name_regex + r"$", + r"^" + _hostname_regex + r"$", ), - ( - "EMAIL_ADDRESS", - (r"^" + _email_regex + r"$",), - ), - ( - "OPEN_TCP_PORT", - tuple(r"^" + r + r"$" for r in _open_port_regexes), - ), - ( - "URL", - tuple(r"^" + r + r"$" for r in _url_regexes), - ), - ) + ), + ( + "EMAIL_ADDRESS", + (r"^" + _email_regex + r"$",), + ), + ( + "OPEN_TCP_PORT", + tuple(r"^" + r + r"$" for r in _open_port_regexes), + ), + ( + "URL", + tuple(r"^" + r + r"$" for r in _url_regexes), + ), + ) ) ) event_id_regex = re.compile(r"[0-9a-f]{40}:[A-Z0-9_]+") scan_name_regex = re.compile(r"[a-z]{3,20}_[a-z]{3,20}") - # For use with extract_params_html helper input_tag_regex = re.compile(r"]+?name=[\"\'](\w+)[\"\']") jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") diff --git a/bbot/core/helpers/url.py b/bbot/core/helpers/url.py index 5482e54c5..30a5867ed 100644 --- a/bbot/core/helpers/url.py +++ b/bbot/core/helpers/url.py @@ -5,7 +5,6 @@ from .regexes import double_slash_regex - log = logging.getLogger("bbot.core.helpers.url") @@ -27,9 +26,7 @@ def parse_url(url): >>> parse_url('https://www.evilcorp.com') ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='', fragment='') """ - if isinstance(url, ParseResult): - return url - return urlparse(url) + return url if isinstance(url, ParseResult) else urlparse(url) def add_get_params(url, params): @@ -55,7 +52,7 @@ def add_get_params(url, params): """ parsed = parse_url(url) old_params = dict(parse_qs(parsed.query)) - old_params.update(params) + old_params |= params return parsed._replace(query=urlencode(old_params, doseq=True)) @@ -183,8 +180,7 @@ def hash_url(url): parsed = parsed._replace(fragment="", query="") to_hash = [parsed.netloc] for segment in parsed.path.split("/"): - hash_segment = [] - hash_segment.append(charset(segment)) + hash_segment = [charset(segment)] hash_segment.append(param_type(segment)) dot_split = segment.split(".") if len(dot_split) > 1: diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py index 5b0dfc284..08ad64073 100644 --- a/bbot/core/helpers/validators.py +++ b/bbot/core/helpers/validators.py @@ -142,7 +142,14 @@ def validate_url_parsed(url): @validator def validate_severity(severity): severity = str(severity).strip().upper() - if not severity in ("UNKNOWN", "INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"): + if severity not in { + "UNKNOWN", + "INFO", + "LOW", + "MEDIUM", + "HIGH", + "CRITICAL", + }: raise ValueError(f"Invalid severity: {severity}") return severity @@ -236,7 +243,7 @@ def collapse_urls(urls, threshold=10): new_url, } - for url_hash, new_urls in url_hashes.items(): + for new_urls in url_hashes.values(): # if the number of URLs exceeds the threshold if len(new_urls) > threshold: # yield only one @@ -270,8 +277,8 @@ def soft_validate(s, t): """ try: validator_fn = globals()[f"validate_{t.strip().lower()}"] - except KeyError: - raise ValueError(f'No validator for type "{t}"') + except KeyError as e: + raise ValueError(f'No validator for type "{t}"') from e try: validator_fn(s) return True diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index accd55950..61e9bf858 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -52,15 +52,14 @@ def __init__(self, *args, **kwargs): web_requests_per_second = self._bbot_scan.config.get("web_requests_per_second", 100) self._rate_limiter = RateLimiter(web_requests_per_second, "Web") - http_debug = self._bbot_scan.config.get("http_debug", None) - if http_debug: + if http_debug := self._bbot_scan.config.get("http_debug", None): log.debug(f"Creating AsyncClient: {args}, {kwargs}") self._persist_cookies = kwargs.pop("persist_cookies", True) # timeout http_timeout = self._bbot_scan.config.get("http_timeout", 20) - if not "timeout" in kwargs: + if "timeout" not in kwargs: kwargs["timeout"] = http_timeout # headers @@ -95,9 +94,7 @@ def build_request(self, *args, **kwargs): return request def _merge_cookies(self, cookies): - if self._persist_cookies: - return super()._merge_cookies(cookies) - return cookies + return super()._merge_cookies(cookies) if self._persist_cookies else cookies class WebHelper: @@ -222,7 +219,7 @@ async def request(self, *args, **kwargs): async with self._acatch(url, raise_error): if self.http_debug: - logstr = f"Web request: {str(args)}, {str(kwargs)}" + logstr = f"Web request: {str(args)}, {kwargs}" log.debug(logstr) response = await client.request(*args, **kwargs) if self.http_debug: @@ -263,8 +260,6 @@ async def download(self, url, **kwargs): if max_size is not None: max_size = self.parent_helper.human_to_bytes(max_size) cache_hrs = float(kwargs.pop("cache_hrs", -1)) - total_size = 0 - chunk_size = 8192 log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}") if cache_hrs > 0 and self.parent_helper.is_cached(url): log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}") @@ -273,8 +268,10 @@ async def download(self, url, **kwargs): # kwargs["raise_error"] = True # kwargs["stream"] = True kwargs["follow_redirects"] = follow_redirects - if not "method" in kwargs: + if "method" not in kwargs: kwargs["method"] = "GET" + total_size = 0 + chunk_size = 8192 try: async with self._acatch(url, raise_error), self.AsyncClient().stream(url=url, **kwargs) as response: status_code = getattr(response, "status_code", 0) @@ -294,9 +291,7 @@ async def download(self, url, **kwargs): f.write(chunk) success = True except httpx.HTTPError as e: - log_fn = log.verbose - if warn: - log_fn = log.warning + log_fn = log.warning if warn else log.verbose log_fn(f"Failed to download {url}: {e}") return @@ -332,7 +327,7 @@ async def wordlist(self, path, lines=None, **kwargs): """ if not path: raise WordlistError(f"Invalid wordlist: {path}") - if not "cache_hrs" in kwargs: + if "cache_hrs" not in kwargs: kwargs["cache_hrs"] = 720 if self.parent_helper.is_url(path): filename = await self.download(str(path), **kwargs) @@ -345,16 +340,15 @@ async def wordlist(self, path, lines=None, **kwargs): if lines is None: return filename - else: - lines = int(lines) - with open(filename) as f: - read_lines = f.readlines() - cache_key = f"{filename}:{lines}" - truncated_filename = self.parent_helper.cache_filename(cache_key) - with open(truncated_filename, "w") as f: - for line in read_lines[:lines]: - f.write(line) - return truncated_filename + lines = int(lines) + with open(filename) as f: + read_lines = f.readlines() + cache_key = f"{filename}:{lines}" + truncated_filename = self.parent_helper.cache_filename(cache_key) + with open(truncated_filename, "w") as f: + for line in read_lines[:lines]: + f.write(line) + return truncated_filename async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs): """ @@ -451,8 +445,7 @@ async def curl(self, *args, **kwargs): curl_command = ["curl", url, "-s"] - raw_path = kwargs.get("raw_path", False) - if raw_path: + if raw_path := kwargs.get("raw_path", False): curl_command.append("--path-as-is") # respect global ssl verify settings @@ -461,9 +454,9 @@ async def curl(self, *args, **kwargs): headers = kwargs.get("headers", {}) - ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False) - - if ignore_bbot_global_settings: + if ignore_bbot_global_settings := kwargs.get( + "ignore_bbot_global_settings", False + ): log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied") else: http_timeout = self.parent_helper.config.get("http_timeout", 20) @@ -478,7 +471,7 @@ async def curl(self, *args, **kwargs): headers[hk] = hv # add the timeout - if not "timeout" in kwargs: + if "timeout" not in kwargs: timeout = http_timeout curl_command.append("-m") @@ -487,50 +480,36 @@ async def curl(self, *args, **kwargs): for k, v in headers.items(): if isinstance(v, list): for x in v: - curl_command.append("-H") - curl_command.append(f"{k}: {x}") - + curl_command.extend(("-H", f"{k}: {x}")) else: - curl_command.append("-H") - curl_command.append(f"{k}: {v}") - + curl_command.extend(("-H", f"{k}: {v}")) post_data = kwargs.get("post_data", {}) if len(post_data.items()) > 0: curl_command.append("-d") - post_data_str = "" - for k, v in post_data.items(): - post_data_str += f"&{k}={v}" + post_data_str = "".join(f"&{k}={v}" for k, v in post_data.items()) curl_command.append(post_data_str.lstrip("&")) - method = kwargs.get("method", "") - if method: + if method := kwargs.get("method", ""): curl_command.append("-X") curl_command.append(method) - cookies = kwargs.get("cookies", "") - if cookies: + if cookies := kwargs.get("cookies", ""): curl_command.append("-b") - cookies_str = "" - for k, v in cookies.items(): - cookies_str += f"{k}={v}; " + cookies_str = "".join(f"{k}={v}; " for k, v in cookies.items()) curl_command.append(f'{cookies_str.rstrip(" ")}') - path_override = kwargs.get("path_override", None) - if path_override: + if path_override := kwargs.get("path_override", None): curl_command.append("--request-target") curl_command.append(f"{path_override}") - head_mode = kwargs.get("head_mode", None) - if head_mode: + if head_mode := kwargs.get("head_mode", None): curl_command.append("-I") - raw_body = kwargs.get("raw_body", None) - if raw_body: + if raw_body := kwargs.get("raw_body", None): curl_command.append("-d") curl_command.append(raw_body) - output = (await self.parent_helper.run(curl_command)).stdout - return output + return (await self.parent_helper.run(curl_command)).stdout def is_spider_danger(self, source_event, url): """ @@ -561,9 +540,7 @@ def is_spider_danger(self, source_event, url): web_spider_depth = self.parent_helper.scan.config.get("web_spider_depth", 1) spider_distance = getattr(source_event, "web_spider_distance", 0) + 1 web_spider_distance = self.parent_helper.scan.config.get("web_spider_distance", 0) - if (url_depth > web_spider_depth) or (spider_distance > web_spider_distance): - return True - return False + return url_depth > web_spider_depth or spider_distance > web_spider_distance def ssl_context_noverify(self): if self._ssl_context_noverify is None: @@ -608,13 +585,13 @@ async def _acatch(self, url, raise_error): log.trace(msg) log.trace(traceback.format_exc()) if raise_error: - raise httpx.RequestError(msg) + raise httpx.RequestError(msg) from e except anyio.EndOfStream as e: msg = f"AnyIO error with request to URL: {url}: {e}" log.trace(msg) log.trace(traceback.format_exc()) if raise_error: - raise httpx.RequestError(msg) + raise httpx.RequestError(msg) from e except BaseException as e: log.trace(f"Unhandled exception with request to URL: {url}: {e}") log.trace(traceback.format_exc()) diff --git a/bbot/core/helpers/wordcloud.py b/bbot/core/helpers/wordcloud.py index 7531a93e2..62d13d60d 100644 --- a/bbot/core/helpers/wordcloud.py +++ b/bbot/core/helpers/wordcloud.py @@ -87,7 +87,7 @@ def __init__(self, parent_helper, *args, **kwargs): super().__init__(*args, **kwargs) def mutations( - self, words, devops=True, cloud=True, letters=True, numbers=5, number_padding=2, substitute_numbers=True + self, words, devops=True, cloud=True, letters=True, numbers=5, number_padding=2, substitute_numbers=True ): """ Generate various mutations for the given list of words based on different criteria. @@ -111,20 +111,19 @@ def mutations( results = set() for word in words: h = hash(word) - if not h in results: + if h not in results: results.add(h) yield (word,) - if numbers > 0: - if substitute_numbers: - for word in words: - for number_mutation in self.get_number_mutations(word, n=numbers, padding=number_padding): - h = hash(number_mutation) - if not h in results: - results.add(h) - yield (number_mutation,) + if numbers > 0 and substitute_numbers: + for word in words: + for number_mutation in self.get_number_mutations(word, n=numbers, padding=number_padding): + h = hash(number_mutation) + if h not in results: + results.add(h) + yield (number_mutation,) for word in words: for modifier in self.modifiers( - devops=devops, cloud=cloud, letters=letters, numbers=numbers, number_padding=number_padding + devops=devops, cloud=cloud, letters=letters, numbers=numbers, number_padding=number_padding ): a = (word, modifier) b = (modifier, word) @@ -251,8 +250,8 @@ def get_number_mutations(self, base, n=5, padding=2): for match in list(self.parent_helper.regexes.num_regex.finditer(base))[-3:]: span = match.span() before = base[: span[0]] - after = base[span[-1] :] - number = base[span[0] : span[-1]] + after = base[span[-1]:] + number = base[span[0]: span[-1]] numlen = len(number) maxnum = min(int("9" * numlen), int(number) + n) minnum = max(0, int(number) - n) @@ -272,7 +271,7 @@ def get_number_mutations(self, base, n=5, padding=2): span = match.span() for suffix in number_suffixes: before = base[: span[-1]] - after = base[span[-1] :] + after = base[span[-1]:] # skip if there's already a number if len(after) > 1 and not after[0].isdigit(): results.add(f"{before}{suffix}{after}") @@ -322,7 +321,7 @@ def json(self, limit=None): @property def default_filename(self): - return self.parent_helper.scan.home / f"wordcloud.tsv" + return self.parent_helper.scan.home / "wordcloud.tsv" def save(self, filename=None, limit=None): """ @@ -357,7 +356,7 @@ def save(self, filename=None, limit=None): log.debug(f"Saved word cloud ({len(self):,} words) to {filename}") return True, filename else: - log.debug(f"No words to save") + log.debug("No words to save") except Exception as e: import traceback @@ -394,9 +393,7 @@ def load(self, filename=None): except Exception as e: import traceback - log_fn = log.debug - if filename is not None: - log_fn = log.warning + log_fn = log.warning if filename is not None else log.debug log_fn(f"Failed to load word cloud from {wordcloud_path}: {e}") if filename is not None: log.trace(traceback.format_exc()) @@ -439,7 +436,7 @@ def top_mutations(self, n=None): def _add_mutation(self, mutation): if None not in mutation: return - mutation = tuple([m for m in mutation if m != ""]) + mutation = tuple(m for m in mutation if m != "") try: self[mutation] += 1 except KeyError: @@ -519,7 +516,7 @@ def add_word(self, word): if s.isdigit(): continue split_before = "".join(match_str_split[:i]) - split_after = "".join(match_str_split[i + 1 :]) + split_after = "".join(match_str_split[i + 1:]) wordninja_mutation = (before + split_before, None, split_after + after) mutations.add(wordninja_mutation) for m in mutations: diff --git a/bbot/core/logger/logger.py b/bbot/core/logger/logger.py index a2c12b189..2494f93e4 100644 --- a/bbot/core/logger/logger.py +++ b/bbot/core/logger/logger.py @@ -9,7 +9,6 @@ from ..helpers.misc import mkdir, error_and_exit from ..helpers.logger import colorize, loglevel_mapping - _log_level_override = None bbot_loggers = None @@ -101,7 +100,6 @@ def logToRoot(message, *args, **kwargs): addLoggingLevel("HUGEVERBOSE", 16) addLoggingLevel("VERBOSE", 15) - verbosity_levels_toggle = [logging.INFO, logging.VERBOSE, logging.DEBUG] @@ -167,9 +165,7 @@ def stderr_filter(record): log_level = get_log_level() if record.levelno == logging.STDOUT or (record.levelno == logging.TRACE and log_level > logging.DEBUG): return False - if record.levelno < log_level: - return False - return True + return record.levelno >= log_level # Log to stderr stderr_handler = logging.StreamHandler(sys.stderr) diff --git a/bbot/db/neo4j.py b/bbot/db/neo4j.py index 7d718d64b..c2f9ddce2 100644 --- a/bbot/db/neo4j.py +++ b/bbot/db/neo4j.py @@ -4,6 +4,7 @@ log = logging.getLogger("bbot.db.neo4j") + # uncomment this to enable neo4j debugging # logging.basicConfig(level=logging.DEBUG, format="%(message)s") diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index 7b0cda171..06d3c2e8a 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -20,20 +20,17 @@ def abort_if_pre(self, hostname): This exists because of the _disgusting_ amount of garbage data in this API """ dns_depth = hostname.count(".") + 1 - if dns_depth > self.dns_abort_depth: - return True - return False + return dns_depth > self.dns_abort_depth async def abort_if(self, event): # abort if dns name is unresolved - if not "resolved" in event.tags: + if "resolved" not in event.tags: return True, "DNS name is unresolved" return await super().abort_if(event) def parse_results(self, r, query): results = set() - json = r.json() - if json: + if json := r.json(): for hostname in json: hostname = str(hostname).lower() if hostname.endswith(f".{query}") and not self.abort_if_pre(hostname): diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py index 4fcf9d7d9..bf9505381 100644 --- a/bbot/modules/azure_tenant.py +++ b/bbot/modules/azure_tenant.py @@ -25,8 +25,9 @@ async def handle_event(self, event): tenant_id = None authorization_endpoint = openid_config.get("authorization_endpoint", "") - matches = self.helpers.regexes.uuid_regex.findall(authorization_endpoint) - if matches: + if matches := self.helpers.regexes.uuid_regex.findall( + authorization_endpoint + ): tenant_id = matches[0] tenant_names = set() @@ -37,8 +38,7 @@ async def handle_event(self, event): self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate", "azure-tenant"]) # tenant names if domain.lower().endswith(".onmicrosoft.com"): - tenantname = domain.split(".")[0].lower() - if tenantname: + if tenantname := domain.split(".")[0].lower(): tenant_names.add(tenantname) event_data = {"tenant-names": sorted(tenant_names), "domains": sorted(domains)} diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index cc52feecb..ebd5337a7 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -21,8 +21,7 @@ async def handle_event(self, event): resp_headers = event.data.get("header", None) resp_cookies = {} if resp_headers: - resp_cookies_raw = resp_headers.get("set_cookie", None) - if resp_cookies_raw: + if resp_cookies_raw := resp_headers.get("set_cookie", None): if "," in resp_cookies_raw: resp_cookies_list = resp_cookies_raw.split(",") else: diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 125b9c14b..a239a4468 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -258,15 +258,14 @@ async def require_api_key(self): - Sets the API key readiness status accordingly. """ self.api_key = self.config.get("api_key", "") - if self.auth_secret: - try: - await self.ping() - self.hugesuccess(f"API is ready") - return True - except Exception as e: - return None, f"Error with API ({str(e).strip()})" - else: + if not self.auth_secret: return None, "No API key set" + try: + await self.ping() + self.hugesuccess("API is ready") + return True + except Exception as e: + return None, f"Error with API ({str(e).strip()})" async def ping(self): """Asynchronously checks the health of the configured API. @@ -430,8 +429,7 @@ def emit_event(self, *args, **kwargs): v = event_kwargs.pop(o, None) if v is not None: emit_kwargs[o] = v - event = self.make_event(*args, **event_kwargs) - if event: + if event := self.make_event(*args, **event_kwargs): self.queue_outgoing_event(event, **emit_kwargs) async def emit_event_wait(self, *args, **kwargs): @@ -474,9 +472,7 @@ async def _events_waiting(self): """ events = [] finish = False - while self.incoming_event_queue: - if len(events) > self.batch_size: - break + while self.incoming_event_queue and len(events) <= self.batch_size: try: event = self.incoming_event_queue.get_nowait() self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") @@ -495,10 +491,11 @@ async def _events_waiting(self): @property def num_incoming_events(self): - ret = 0 - if self.incoming_event_queue is not False: - ret = self.incoming_event_queue.qsize() - return ret + return ( + self.incoming_event_queue.qsize() + if self.incoming_event_queue is not False + else 0 + ) def start(self): self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] @@ -586,7 +583,7 @@ async def _worker(self): if self.incoming_event_queue is not False: event = await self.incoming_event_queue.get() else: - self.debug(f"Event queue is in bad state") + self.debug("Event queue is in bad state") break except asyncio.queues.QueueEmpty: continue @@ -610,7 +607,7 @@ async def _worker(self): except asyncio.CancelledError: self.log.trace("Worker cancelled") raise - self.log.trace(f"Worker stopped") + self.log.trace("Worker stopped") @property def max_scope_distance(self): @@ -651,29 +648,24 @@ def _event_precheck(self, event): if event.type in ("FINISHED",): return True, "its type is FINISHED" if self.errored: - return False, f"module is in error state" + return False, "module is in error state" # exclude non-watched types - if not any(t in self.get_watched_events() for t in ("*", event.type)): + if all(t not in self.get_watched_events() for t in ("*", event.type)): return False, "its type is not in watched_events" - if self.target_only: - if "target" not in event.tags: - return False, "it did not meet target_only filter criteria" + if self.target_only and "target" not in event.tags: + return False, "it did not meet target_only filter criteria" # exclude certain URLs (e.g. javascript): if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: return False, "its extension was listed in url_extension_httpx_only" # if event is an IP address that was speculated from a CIDR source_is_range = getattr(event.source, "type", "") == "IP_RANGE" if ( - source_is_range - and event.type == "IP_ADDRESS" - and str(event.module) == "speculate" - and self.name != "speculate" - ): - # and the current module listens for both ranges and CIDRs - if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]): - # then skip the event. - # this helps avoid double-portscanning both an individual IP and its parent CIDR. - return False, "module consumes IP ranges directly" + source_is_range + and event.type == "IP_ADDRESS" + and str(event.module) == "speculate" + and self.name != "speculate" + ) and all(x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")): + return False, "module consumes IP ranges directly" return True, "precheck succeeded" async def _event_postcheck(self, event): @@ -748,9 +740,8 @@ async def _event_postcheck(self, event): return True, "" def _scope_distance_check(self, event): - if self.in_scope_only: - if event.scope_distance > 0: - return False, "it did not meet in_scope_only filter criteria" + if self.in_scope_only and event.scope_distance > 0: + return False, "it did not meet in_scope_only filter criteria" if self.scope_distance_modifier is not None: if event.scope_distance < 0: return False, f"its scope_distance ({event.scope_distance}) is invalid." @@ -765,8 +756,8 @@ async def _cleanup(self): if not self._cleanedup: self._cleanedup = True for callback in [self.cleanup] + self.cleanup_callbacks: - context = f"{self.name}.cleanup()" if callable(callback): + context = f"{self.name}.cleanup()" async with self.scan._acatch(context), self._task_counter.count(context): await self.helpers.execute_sync_or_async(callback) @@ -791,7 +782,7 @@ async def queue_event(self, event): """ async with self._task_counter.count("queue_event()", _log=False): if self.incoming_event_queue is False: - self.debug(f"Not in an acceptable state to queue incoming event") + self.debug("Not in an acceptable state to queue incoming event") return acceptable, reason = self._event_precheck(event) if not acceptable: @@ -807,7 +798,7 @@ async def queue_event(self, event): if event.type != "FINISHED": self.scan.manager._new_activity = True except AttributeError: - self.debug(f"Not in an acceptable state to queue incoming event") + self.debug("Not in an acceptable state to queue incoming event") def queue_outgoing_event(self, event, **kwargs): """ @@ -832,7 +823,7 @@ def queue_outgoing_event(self, event, **kwargs): try: self.outgoing_event_queue.put_nowait((event, kwargs)) except AttributeError: - self.debug(f"Not in an acceptable state to queue outgoing event") + self.debug("Not in an acceptable state to queue outgoing event") def set_error_state(self, message=None, clear_outgoing_queue=False): """ @@ -855,26 +846,27 @@ def set_error_state(self, message=None, clear_outgoing_queue=False): - The function sets `self._incoming_event_queue` to False to prevent its further use. - If the module was already in an errored state, the function will not reset the error state or the queue. """ - if not self.errored: - log_msg = f"Setting error state for module {self.name}" - if message is not None: - log_msg += f": {message}" - self.warning(log_msg) - self.errored = True - # clear incoming queue - if self.incoming_event_queue is not False: - self.debug(f"Emptying event_queue") - with suppress(asyncio.queues.QueueEmpty): - while 1: - self.incoming_event_queue.get_nowait() - # set queue to None to prevent its use - # if there are leftover objects in the queue, the scan will hang. - self._incoming_event_queue = False - - if clear_outgoing_queue: - with suppress(asyncio.queues.QueueEmpty): - while 1: - self.outgoing_event_queue.get_nowait() + if self.errored: + return + log_msg = f"Setting error state for module {self.name}" + if message is not None: + log_msg += f": {message}" + self.warning(log_msg) + self.errored = True + # clear incoming queue + if self.incoming_event_queue is not False: + self.debug("Emptying event_queue") + with suppress(asyncio.queues.QueueEmpty): + while 1: + self.incoming_event_queue.get_nowait() + # set queue to None to prevent its use + # if there are leftover objects in the queue, the scan will hang. + self._incoming_event_queue = False + + if clear_outgoing_queue: + with suppress(asyncio.queues.QueueEmpty): + while 1: + self.outgoing_event_queue.get_nowait() def get_per_host_hash(self, event): """ @@ -949,13 +941,15 @@ def status(self): >>> self.status {'events': {'incoming': 5, 'outgoing': 2}, 'tasks': 3, 'errored': False, 'running': True} """ - status = { - "events": {"incoming": self.num_incoming_events, "outgoing": self.outgoing_event_queue.qsize()}, + return { + "events": { + "incoming": self.num_incoming_events, + "outgoing": self.outgoing_event_queue.qsize(), + }, "tasks": self._task_counter.value, "errored": self.errored, + "running": self.running, } - status["running"] = self.running - return status @property def running(self): diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 0b54d40f4..651633f5b 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -46,15 +46,14 @@ async def request_urls(self, query): return await self.request_with_fail_count(url, headers=self.headers) def parse_subdomains(self, r, query=None): - results = set() - subdomains = r.json().get("subdomains") - if subdomains: - results.update(subdomains) - return results + return self._extracted_from_parse_urls_2(r, "subdomains") def parse_urls(self, r, query=None): + return self._extracted_from_parse_urls_2(r, "urls") + + # TODO Rename this here and in `parse_subdomains` and `parse_urls` + def _extracted_from_parse_urls_2(self, r, arg1): results = set() - urls = r.json().get("urls") - if urls: - results.update(urls) + if subdomains := r.json().get(arg1): + results.update(subdomains) return results diff --git a/bbot/modules/bucket_aws.py b/bbot/modules/bucket_aws.py index 5aa9524b2..e11ce4f2c 100644 --- a/bbot/modules/bucket_aws.py +++ b/bbot/modules/bucket_aws.py @@ -27,9 +27,8 @@ async def setup(self): async def filter_event(self, event): if event.type == "DNS_NAME" and event.scope_distance > 0: return False, "only accepts in-scope DNS_NAMEs" - if event.type == "STORAGE_BUCKET": - if f"cloud-{self.cloud_helper_name}" not in event.tags: - return False, "bucket belongs to a different cloud provider" + if f"cloud-{self.cloud_helper_name}" not in event.tags and event.type == "STORAGE_BUCKET": + return False, "bucket belongs to a different cloud provider" return True async def handle_event(self, event): @@ -59,7 +58,7 @@ async def handle_storage_bucket(self, event): self.emit_event(event_data, "FINDING", source=event, tags=tags) async for bucket_name, url, tags in self.brute_buckets( - [bucket_name], permutations=self.permutations, omit_base=True + [bucket_name], permutations=self.permutations, omit_base=True ): self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) @@ -106,16 +105,14 @@ async def check_bucket_open(self, bucket_name, url): status_code = getattr(response, "status_code", 404) content = getattr(response, "text", "") open_bucket = status_code == 200 and "Contents" in content - msg = "" - if open_bucket: - msg = "Open storage bucket" + msg = "Open storage bucket" if open_bucket else "" return (msg, tags) def valid_bucket_name(self, bucket_name): valid = self.cloud_helper.is_valid_bucket(bucket_name) if valid and not self.helpers.is_ip(bucket_name): bucket_hash = hash(bucket_name) - if not bucket_hash in self.buckets_tried: + if bucket_hash not in self.buckets_tried: self.buckets_tried.add(bucket_hash) return True return False diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index bda49a3a4..b7361d91d 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -24,7 +24,5 @@ async def check_bucket_open(self, bucket_name, url): response = await self.helpers.request(url) tags = self.gen_tags_exists(response) status_code = getattr(response, "status_code", 404) - msg = "" - if status_code == 200: - msg = "Open storage bucket" + msg = "Open storage bucket" if status_code == 200 else "" return (msg, tags) diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py index 8c1c45674..c394d8c87 100644 --- a/bbot/modules/bucket_gcp.py +++ b/bbot/modules/bucket_gcp.py @@ -32,7 +32,7 @@ async def check_bucket_open(self, bucket_name, url): bad_permissions = [] try: list_permissions = "&".join(["=".join(("permissions", p)) for p in self.bad_permissions]) - url = f"https://www.googleapis.com/storage/v1/b/{bucket_name}/iam/testPermissions?" + list_permissions + url = f"https://www.googleapis.com/storage/v1/b/{bucket_name}/iam/testPermissions?{list_permissions}" response = await self.helpers.request(url) permissions = response.json() if isinstance(permissions, dict): diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 25a46ddf5..bec3170f9 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -63,8 +63,7 @@ def parse_domains(self, r, query): results_set = set() json = r.json() if json and isinstance(json, dict): - results = json.get("Results", []) - if results: + if results := json.get("Results", []): for result in results: for chunk in result.get("Result", {}).get("Paths", []): domain = chunk.get("Domain", "") @@ -73,11 +72,9 @@ def parse_domains(self, r, query): if subdomain: domain = f"{subdomain}.{domain}" results_set.add(domain) - else: - errors = json.get("Errors", [{}]) - if errors: - error = errors[0].get("Message", "Unknown Error") - self.verbose(f"No results for {query}: {error}") + elif errors := json.get("Errors", [{}]): + error = errors[0].get("Message", "Unknown Error") + self.verbose(f"No results for {query}: {error}") return results_set def parse_redirects(self, r, query): @@ -101,16 +98,13 @@ def parse_redirects(self, r, query): outbound = json.get("Outbound", []) if inbound: for i in inbound: - domain = i.get("Domain", "") - if domain: + if domain := i.get("Domain", ""): results.add(domain) if outbound: for o in outbound: - domain = o.get("Domain", "") - if domain: + if domain := o.get("Domain", ""): results.add(domain) if not results: - error = json.get("error", "") - if error: + if error := json.get("error", ""): self.warning(f"No results for {query}: {error}") return results diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py index 98cffd58a..a9ce87a4c 100644 --- a/bbot/modules/bypass403.py +++ b/bbot/modules/bypass403.py @@ -21,7 +21,6 @@ ("GET", "{scheme}://{netloc}/(S(X))/../(S(X))/{path}", None, True), # ASPNET COOKIELESS URLS ] - query_payloads = [ "%09", "%20", @@ -70,8 +69,10 @@ if "?" not in qp: # we only want to use "?" after the path signatures.append(("GET", "{scheme}://{netloc}/%s/{path}" % qp, None, True)) -for hp_key in header_payloads.keys(): - signatures.append(("GET", "{scheme}://{netloc}/{path}", {hp_key: header_payloads[hp_key]}, False)) +signatures.extend( + ("GET", "{scheme}://{netloc}/{path}", {hp_key: value}, False) + for hp_key, value in header_payloads.items() +) class bypass403(BaseModule): @@ -86,10 +87,7 @@ async def do_checks(self, compare_helper, event, collapse_threshold): for sig in signatures: sig = self.format_signature(sig, event) - if sig[2] != None: - headers = dict(sig[2]) - else: - headers = None + headers = dict(sig[2]) if sig[2] != None else None match, reasons, reflection, subject_response = await compare_helper.compare( sig[1], headers=headers, method=sig[0], allow_redirects=True ) @@ -130,7 +128,7 @@ async def handle_event(self, event): if len(results) > collapse_threshold: self.emit_event( { - "description": f"403 Bypass MULTIPLE SIGNATURES (exceeded threshold {str(collapse_threshold)})", + "description": f"403 Bypass MULTIPLE SIGNATURES (exceeded threshold {collapse_threshold})", "host": str(event.host), "url": event.data, }, @@ -146,9 +144,7 @@ async def handle_event(self, event): ) async def filter_event(self, event): - if ("status-403" in event.tags) or ("status-401" in event.tags): - return True - return False + return "status-403" in event.tags or "status-401" in event.tags def format_signature(self, sig, event): if sig[3] == True: diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 8e05a1c4b..3ce4d03da 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -23,9 +23,7 @@ async def request_url(self, query): def parse_results(self, r, query): j = r.json() if isinstance(j, dict): - subdomains = j.get("subdomains", []) - if subdomains: + if subdomains := j.get("subdomains", []): for s in subdomains: - subdomain = s.get("subdomain", "") - if subdomain: + if subdomain := s.get("subdomain", ""): yield subdomain diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py index 339f10bf7..a77e7df24 100644 --- a/bbot/modules/censys.py +++ b/bbot/modules/censys.py @@ -46,7 +46,7 @@ async def query(self, query): "per_page": 100, } if cursor: - json_data.update({"cursor": cursor}) + json_data |= {"cursor": cursor} resp = await self.helpers.request( url, method="POST", @@ -61,16 +61,13 @@ async def query(self, query): if resp.status_code < 200 or resp.status_code >= 400: if isinstance(d, dict): - error = d.get("error", "") - if error: + if error := d.get("error", ""): self.warning(error) - self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i+1}') + self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i + 1}') self.debug(f"Response: {resp.text}") break else: - if d is None: - break - elif not isinstance(d, dict): + if d is None or not isinstance(d, dict): break status = d.get("status", "").lower() result = d.get("result", {}) diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py index 4441b9d98..dfa7f3298 100644 --- a/bbot/modules/certspotter.py +++ b/bbot/modules/certspotter.py @@ -14,8 +14,7 @@ def request_url(self, query): return self.request_with_fail_count(url, timeout=self.http_timeout + 30) def parse_results(self, r, query): - json = r.json() - if json: + if json := r.json(): for r in json: for dns_name in r.get("dns_names", []): yield dns_name.lstrip(".*").rstrip(".") diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index 3eb763573..e6c04afea 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -23,11 +23,10 @@ async def request_url(self, query): def parse_results(self, r, query): j = r.json() - subdomains_set = set() if isinstance(j, dict): - domain = j.get("domain", "") - if domain: + if domain := j.get("domain", ""): subdomains = j.get("subdomains", []) + subdomains_set = set() for s in subdomains: s = s.lower().strip(".*") subdomains_set.add(s) diff --git a/bbot/modules/columbus.py b/bbot/modules/columbus.py index 2e8901359..b5b9f6258 100644 --- a/bbot/modules/columbus.py +++ b/bbot/modules/columbus.py @@ -14,8 +14,8 @@ async def request_url(self, query): return await self.request_with_fail_count(url) def parse_results(self, r, query): - results = set() json = r.json() if json and isinstance(json, list): - return set([f"{s.lower()}.{query}" for s in json]) - return results + return {f"{s.lower()}.{query}" for s in json} + else: + return set() diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py index 9773f72d4..c1af0009f 100644 --- a/bbot/modules/crt.py +++ b/bbot/modules/crt.py @@ -22,13 +22,11 @@ async def request_url(self, query): def parse_results(self, r, query): j = r.json() for cert_info in j: - if not type(cert_info) == dict: + if type(cert_info) != dict: continue - cert_id = cert_info.get("id") - if cert_id: + if cert_id := cert_info.get("id"): if hash(cert_id) not in self.cert_ids: self.cert_ids.add(hash(cert_id)) - domain = cert_info.get("name_value") - if domain: + if domain := cert_info.get("name_value"): for d in domain.splitlines(): yield d.lower() diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index 0dad1b63f..5a7cb6c17 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -48,7 +48,7 @@ class ffuf(BaseModule): in_scope_only = True async def setup(self): - self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) + self.canary = "".join(random.choice(string.ascii_lowercase) for _ in range(10)) wordlist_url = self.config.get("wordlist", "") self.debug(f"Using wordlist [{wordlist_url}]") self.wordlist = await self.helpers.wordlist(wordlist_url) @@ -65,7 +65,7 @@ async def setup(self): async def handle_event(self, event): if self.helpers.url_depth(event.data) > self.config.get("max_depth"): - self.debug(f"Exceeded max depth, aborting event") + self.debug("Exceeded max depth, aborting event") return # only FFUF against a directory @@ -78,9 +78,7 @@ async def handle_event(self, event): exts = ["", "/"] if self.extensions: - for ext in self.extensions: - exts.append(f".{ext}") - + exts.extend(f".{ext}" for ext in self.extensions) filters = await self.baseline_ffuf(fixed_url, exts=exts) async for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters): self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) @@ -91,7 +89,9 @@ async def filter_event(self, event): return False return True - async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal"): + async def baseline_ffuf(self, url, exts=None, prefix="", suffix="", mode="normal"): + if exts is None: + exts = [""] filters = {} for ext in exts: self.debug(f"running baseline for URL [{url}] with ext [{ext}]") @@ -101,20 +101,25 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal canary_length = 4 canary_list = [] - for i in range(0, 4): - canary_list.append("".join(random.choice(string.ascii_lowercase) for i in range(canary_length))) + for _ in range(4): + canary_list.append( + "".join( + random.choice(string.ascii_lowercase) + for _ in range(canary_length) + ) + ) canary_length += 2 canary_temp_file = self.helpers.tempfile(canary_list, pipe=False) async for canary_r in self.execute_ffuf( - canary_temp_file, - url, - prefix=prefix, - suffix=suffix, - mode=mode, - baseline=True, - apply_filters=False, - filters=filters, + canary_temp_file, + url, + prefix=prefix, + suffix=suffix, + mode=mode, + baseline=True, + apply_filters=False, + filters=filters, ): canary_results.append(canary_r) @@ -129,7 +134,7 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal continue # if the codes are different, we should abort, this should also be a warning, as it is highly unusual behavior - if len(set(d["status"] for d in canary_results)) != 1: + if len({d["status"] for d in canary_results}) != 1: self.warning("Got different codes for each baseline. This could indicate load balancing") filters[ext] = ["ABORT", "BASELINE_CHANGED_CODES"] continue @@ -155,7 +160,7 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal continue # we start by seeing if all of the baselines have the same character count - if len(set(d["length"] for d in canary_results)) == 1: + if len({d["length"] for d in canary_results}) == 1: self.debug("All baseline results had the same char count, we can make a filter on that") filters[ext] = [ "-fc", @@ -168,7 +173,7 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal continue # if that doesn't work we can try words - if len(set(d["words"] for d in canary_results)) == 1: + if len({d["words"] for d in canary_results}) == 1: self.debug("All baseline results had the same word count, we can make a filter on that") filters[ext] = [ "-fc", @@ -181,7 +186,7 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal continue # as a last resort we will try lines - if len(set(d["lines"] for d in canary_results)) == 1: + if len({d["lines"] for d in canary_results}) == 1: self.debug("All baseline results had the same word count, we can make a filter on that") filters[ext] = [ "-fc", @@ -198,18 +203,11 @@ async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal return filters - async def execute_ffuf( - self, - tempfile, - url, - prefix="", - suffix="", - exts=[""], - filters={}, - mode="normal", - apply_filters=True, - baseline=False, - ): + async def execute_ffuf(self, tempfile, url, prefix="", suffix="", exts=None, filters=None, mode="normal", apply_filters=True, baseline=False): + if exts is None: + exts = [""] + if filters is None: + filters = {} for ext in exts: if mode == "normal": self.debug("in mode [normal]") @@ -256,10 +254,7 @@ async def execute_ffuf( self.warning(f"Exiting from FFUF run early, received an ABORT filter: [{filters[ext][1]}]") continue - elif filters[ext] == None: - pass - - else: + elif filters[ext] != None: command += filters[ext] else: command.append("-mc") @@ -281,34 +276,38 @@ async def execute_ffuf( self.debug("Found canary! aborting...") return else: - if mode == "normal": - # before emitting, we are going to send another baseline. This will immediately catch things like a WAF flipping blocking on us mid-scan - if baseline == False: - pre_emit_temp_canary = [ - f - async for f in self.execute_ffuf( - self.helpers.tempfile( - ["".join(random.choice(string.ascii_lowercase) for i in range(4))], - pipe=False, - ), - url, - prefix=prefix, - suffix=suffix, - mode=mode, - exts=[ext], - baseline=True, - filters=filters, - ) - ] - if len(pre_emit_temp_canary) == 0: - yield found_json - else: - self.warning( - "Baseline changed mid-scan. This is probably due to a WAF turning on a block against you." - ) - self.warning(f"Aborting the current run against [{url}]") - return - + if mode == "normal" and baseline == False: + if pre_emit_temp_canary := [ + f + async for f in self.execute_ffuf( + self.helpers.tempfile( + [ + "".join( + random.choice( + string.ascii_lowercase + ) + for _ in range(4) + ) + ], + pipe=False, + ), + url, + prefix=prefix, + suffix=suffix, + mode=mode, + exts=[ext], + baseline=True, + filters=filters, + ) + ]: + self.warning( + "Baseline changed mid-scan. This is probably due to a WAF turning on a block against you." + ) + self.warning(f"Aborting the current run against [{url}]") + return + + else: + yield found_json yield found_json except json.decoder.JSONDecodeError: @@ -324,10 +323,12 @@ def generate_templist(self, prefix=None): if len(val) > 0: if val.strip().lower() in self.blacklist: self.debug(f"Skipping adding [{val.strip()}] to wordlist because it was in the blacklist") - else: - if not prefix or val.strip().lower().startswith(prefix.strip().lower()): - if not any(char in val.strip().lower() for char in self.banned_characters): - line_count += 1 - virtual_file.append(f"{val.strip().lower()}") + elif not prefix or val.strip().lower().startswith(prefix.strip().lower()): + if all( + char not in val.strip().lower() + for char in self.banned_characters + ): + line_count += 1 + virtual_file.append(f"{val.strip().lower()}") virtual_file.append(self.canary) return self.helpers.tempfile(virtual_file, pipe=False), line_count diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index df5f86140..c5fd2df36 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -123,7 +123,7 @@ async def setup(self): self.budget_templates_file = self.helpers.tempfile(self.nucleibudget.collapsable_templates, pipe=False) self.info( - f"Loaded [{str(sum(self.nucleibudget.severity_stats.values()))}] templates based on a budget of [{str(self.budget)}] request(s)" + f"Loaded [{str(sum(self.nucleibudget.severity_stats.values()))}] templates based on a budget of [{self.budget}] request(s)" ) self.info( f"Template Severity: Critical [{self.nucleibudget.severity_stats['critical']}] High [{self.nucleibudget.severity_stats['high']}] Medium [{self.nucleibudget.severity_stats['medium']}] Low [{self.nucleibudget.severity_stats['low']}] Info [{self.nucleibudget.severity_stats['info']}] Unknown [{self.nucleibudget.severity_stats['unknown']}]" @@ -199,27 +199,19 @@ async def execute_nuclei(self, nuclei_input): command += ["-r", self.helpers.resolver_file] for cli_option in ("severity", "templates", "iserver", "itoken", "tags", "etags"): - option = getattr(self, cli_option) - - if option: - command.append(f"-{cli_option}") - command.append(option) - + if option := getattr(self, cli_option): + command.extend((f"-{cli_option}", option)) if self.scan.config.get("interactsh_disable") == True: self.info("Disbling interactsh in accordance with global settings") command.append("-no-interactsh") - if self.mode == "technology": - command.append("-as") - if self.mode == "budget": - command.append("-t") - command.append(self.budget_templates_file) + command.extend(("-t", self.budget_templates_file)) + elif self.mode == "technology": + command.append("-as") if self.proxy: - command.append("-proxy") - command.append(f"{self.proxy}") - + command.extend(("-proxy", f"{self.proxy}")) stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status) try: with open(stats_file, "w") as stats_fh: @@ -279,12 +271,11 @@ async def cleanup(self): resume_file.unlink(missing_ok=True) async def filter_event(self, event): - if self.config.get("directory_only", True): - if "endpoint" in event.tags: - self.debug( - f"rejecting URL [{str(event.data)}] because directory_only is true and event has endpoint tag" - ) - return False + if self.config.get("directory_only", True) and "endpoint" in event.tags: + self.debug( + f"rejecting URL [{str(event.data)}] because directory_only is true and event has endpoint tag" + ) + return False return True @@ -307,7 +298,7 @@ def find_budget_paths(self, budget): if yf: for paths in self.get_yaml_request_attr(yf, "path"): for path in paths: - if path in path_frequency.keys(): + if path in path_frequency: path_frequency[path] += 1 else: path_frequency[path] = 1 @@ -321,14 +312,12 @@ def get_yaml_request_attr(self, yf, attr): for r in requests: raw = r.get("raw") if not raw: - res = r.get(attr) - yield res + yield r.get(attr) def get_yaml_info_attr(self, yf, attr): p = self.parse_yaml(yf) info = p.get("info", []) - res = info.get(attr) - yield res + yield info.get(attr) # Parse through all templates and locate those which match the conditions necessary to collapse down to the budget setting def find_collapsable_templates(self): @@ -368,7 +357,7 @@ def find_collapsable_templates(self): collapsable_templates.append(str(yf)) severity_gen = self.get_yaml_info_attr(yf, "severity") severity = next(severity_gen) - if severity in severity_dict.keys(): + if severity in severity_dict: severity_dict[severity] += 1 else: severity_dict[severity] = 1 diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index f4675e10f..e9f9a50ae 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -42,52 +42,55 @@ async def setup(self): return await super().setup() async def handle_event(self, event): - if not self.helpers.is_ip(event.host) or self.config.get("force_basehost"): - host = f"{event.parsed.scheme}://{event.parsed.netloc}" - if host in self.scanned_hosts.keys(): - return - else: - self.scanned_hosts[host] = event - - # subdomain vhost check - self.verbose("Main vhost bruteforce") - if self.config.get("force_basehost"): - basehost = self.config.get("force_basehost") - else: - basehost = self.helpers.parent_domain(event.parsed.netloc) - - self.debug(f"Using basehost: {basehost}") - async for vhost in self.ffuf_vhost(host, f".{basehost}", event): - self.verbose(f"Starting mutations check for {vhost}") - async for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=self.mutations_check(vhost)): - pass + if self.helpers.is_ip(event.host) and not self.config.get( + "force_basehost" + ): + return + host = f"{event.parsed.scheme}://{event.parsed.netloc}" + if host in self.scanned_hosts.keys(): + return + else: + self.scanned_hosts[host] = event + + # subdomain vhost check + self.verbose("Main vhost bruteforce") + if self.config.get("force_basehost"): + basehost = self.config.get("force_basehost") + else: + basehost = self.helpers.parent_domain(event.parsed.netloc) + + self.debug(f"Using basehost: {basehost}") + async for vhost in self.ffuf_vhost(host, f".{basehost}", event): + self.verbose(f"Starting mutations check for {vhost}") + async for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=self.mutations_check(vhost)): + pass - # check existing host for mutations - self.verbose("Checking for vhost mutations on main host") - async for vhost in self.ffuf_vhost( + # check existing host for mutations + self.verbose("Checking for vhost mutations on main host") + async for vhost in self.ffuf_vhost( host, f".{basehost}", event, wordlist=self.mutations_check(event.parsed.netloc.split(".")[0]) - ): - pass + ): + pass - # special vhost list - self.verbose("Checking special vhost list") - async for vhost in self.ffuf_vhost( + # special vhost list + self.verbose("Checking special vhost list") + async for vhost in self.ffuf_vhost( host, "", event, wordlist=self.helpers.tempfile(self.special_vhost_list, pipe=False), skip_dns_host=True, - ): - pass + ): + pass async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=False): filters = await self.baseline_ffuf(f"{host}/", exts=[""], suffix=basehost, mode="hostheader") - self.debug(f"Baseline completed and returned these filters:") + self.debug("Baseline completed and returned these filters:") self.debug(filters) if not wordlist: wordlist = self.tempfile async for r in self.execute_ffuf( - wordlist, host, exts=[""], suffix=basehost, filters=filters, mode="hostheader" + wordlist, host, exts=[""], suffix=basehost, filters=filters, mode="hostheader" ): found_vhost_b64 = r["input"]["FUZZ"] vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()} @@ -101,10 +104,8 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F def mutations_check(self, vhost): mutations_list = [] for mutation in self.helpers.word_cloud.mutations(vhost): - for i in ["", ".", "-"]: - mutations_list.append(i.join(mutation)) - mutations_list_file = self.helpers.tempfile(mutations_list, pipe=False) - return mutations_list_file + mutations_list.extend(i.join(mutation) for i in ["", ".", "-"]) + return self.helpers.tempfile(mutations_list, pipe=False) async def finish(self): # check existing hosts with wordcloud diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index d950514b5..5b52d5c02 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -19,9 +19,7 @@ async def setup(self): return True async def filter_event(self, event): - if any([x in event.tags for x in ("ns-record", "soa-record")]): - return True - return False + return any(x in event.tags for x in ("ns-record", "soa-record")) async def handle_event(self, event): domain = event.data @@ -47,10 +45,7 @@ async def handle_event(self, event): finding_description = f"Successful DNS zone transfer against {nameserver} for {domain}" self.emit_event({"host": str(event.host), "description": finding_description}, "FINDING", source=event) for name, ttl, rdata in zone.iterate_rdatas(): - if str(name) == "@": - parent_data = domain - else: - parent_data = f"{name}.{domain}" + parent_data = domain if str(name) == "@" else f"{name}.{domain}" parent_event = self.make_event(parent_data, "DNS_NAME", event) if not parent_event or parent_event == event: parent_event = event @@ -62,5 +57,4 @@ async def handle_event(self, event): module = self.helpers.dns._get_dummy_module(rdtype) child_event = self.scan.make_event(t, "DNS_NAME", parent_event, module=module) self.emit_event(child_event) - else: - self.debug(f"No data returned by {nameserver} for domain {domain}") + self.debug(f"No data returned by {nameserver} for domain {domain}") diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index d78644e91..3354ee623 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -18,14 +18,12 @@ def find_common_prefixes(strings, minimum_set_length=4): is_substring = False for k, v in frequency_dict.items(): - if prefix != k: - if prefix in k: - is_substring = True + if prefix != k and prefix in k: + is_substring = True if not is_substring: found_prefixes.add(prefix) - else: - if prefix_frequency > v and (len(k) - len(prefix) == 1): - found_prefixes.add(prefix) + elif prefix_frequency > v and (len(k) - len(prefix) == 1): + found_prefixes.add(prefix) return list(found_prefixes) @@ -74,17 +72,13 @@ class ffuf_shortnames(ffuf): in_scope_only = True async def setup(self): - self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) - wordlist = self.config.get("wordlist", "") - if not wordlist: - wordlist = f"{self.helpers.wordlist_dir}/ffuf_shortname_candidates.txt" + self.canary = "".join(random.choice(string.ascii_lowercase) for _ in range(10)) + wordlist = self.config.get("wordlist", "") or f"{self.helpers.wordlist_dir}/ffuf_shortname_candidates.txt" self.debug(f"Using [{wordlist}] for shortname candidate list") self.wordlist = await self.helpers.wordlist(wordlist) self.wordlist_lines = list(self.helpers.read_file(self.wordlist)) - wordlist_extensions = self.config.get("wordlist_extensions", "") - if not wordlist_extensions: - wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt" + wordlist_extensions = self.config.get("wordlist_extensions", "") or f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt" self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list") self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions) @@ -102,17 +96,16 @@ async def setup(self): return True def build_extension_list(self, event): - used_extensions = [] extension_hint = event.parsed.path.rsplit(".", 1)[1].lower().strip() - if len(extension_hint) == 3: - with open(self.wordlist_extensions) as f: - for l in f: - l = l.lower().lstrip(".") - if l.lower().startswith(extension_hint): - used_extensions.append(l.strip()) - return used_extensions - else: + if len(extension_hint) != 3: return [extension_hint] + used_extensions = [] + with open(self.wordlist_extensions) as f: + for l in f: + l = l.lower().lstrip(".") + if l.lower().startswith(extension_hint): + used_extensions.append(l.strip()) + return used_extensions def find_delimeter(self, hint): delimeters = ["_", "-"] @@ -126,116 +119,116 @@ async def filter_event(self, event): return True async def handle_event(self, event): - if event.source.type == "URL": - filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower() + if event.source.type != "URL": + return + filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower() + + host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/" + if host not in self.per_host_collection.keys(): + self.per_host_collection[host] = [(filename_hint, event.source.data)] - host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/" - if host not in self.per_host_collection.keys(): - self.per_host_collection[host] = [(filename_hint, event.source.data)] + else: + self.per_host_collection[host].append((filename_hint, event.source.data)) - else: - self.per_host_collection[host].append((filename_hint, event.source.data)) + self.shortname_to_event[filename_hint] = event - self.shortname_to_event[filename_hint] = event + root_stub = "/".join(event.parsed.path.split("/")[:-1]) + root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/" - root_stub = "/".join(event.parsed.path.split("/")[:-1]) - root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/" + if "shortname-file" in event.tags: + used_extensions = self.build_extension_list(event) + if len(filename_hint) == 6: + tempfile, tempfile_len = self.generate_templist(prefix=filename_hint) + self.verbose( + f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]" + ) + + else: + tempfile = self.helpers.tempfile([filename_hint], pipe=False) + tempfile_len = 1 + + if tempfile_len > 0: if "shortname-file" in event.tags: - used_extensions = self.build_extension_list(event) - - if len(filename_hint) == 6: - tempfile, tempfile_len = self.generate_templist(prefix=filename_hint) - self.verbose( - f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]" - ) - - else: - tempfile = self.helpers.tempfile([filename_hint], pipe=False) - tempfile_len = 1 - - if tempfile_len > 0: - if "shortname-file" in event.tags: - for ext in used_extensions: - async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): - self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) - - elif "shortname-directory" in event.tags: - async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): - r_url = f"{r['url'].rstrip('/')}/" - self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) - - if self.config.get("find_delimeters"): - if "shortname-directory" in event.tags: - delimeter_r = self.find_delimeter(filename_hint) - if delimeter_r: + for ext in used_extensions: + async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): + self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + elif "shortname-directory" in event.tags: + async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): + r_url = f"{r['url'].rstrip('/')}/" + self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + if self.config.get("find_delimeters"): + if "shortname-directory" in event.tags: + if delimeter_r := self.find_delimeter(filename_hint): + delimeter, prefix, partial_hint = delimeter_r + self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]") + tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) + async for r in self.execute_ffuf( + tempfile, root_url, prefix=f"{prefix}{delimeter}", exts=["/"] + ): + self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + + elif "shortname-file" in event.tags: + for ext in used_extensions: + if delimeter_r := self.find_delimeter(filename_hint): delimeter, prefix, partial_hint = delimeter_r self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]") tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) async for r in self.execute_ffuf( - tempfile, root_url, prefix=f"{prefix}{delimeter}", exts=["/"] + tempfile, root_url, prefix=f"{prefix}{delimeter}", suffix=f".{ext}" ): - self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) - - elif "shortname-file" in event.tags: - for ext in used_extensions: - delimeter_r = self.find_delimeter(filename_hint) - if delimeter_r: - delimeter, prefix, partial_hint = delimeter_r - self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]") + self.emit_event( + r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] + ) + + async def finish(self): + if not self.config.get("find_common_prefixes"): + return + per_host_collection = dict(self.per_host_collection) + self.per_host_collection.clear() + + for host, hint_tuple_list in per_host_collection.items(): + hint_list = [x[0] for x in hint_tuple_list] + + common_prefixes = find_common_prefixes(hint_list) + for prefix in common_prefixes: + self.verbose(f"Found common prefix: [{prefix}] for host [{host}]") + for hint_tuple in hint_tuple_list: + hint, url = hint_tuple + if hint.startswith(prefix): + partial_hint = hint[len(prefix):] + + # safeguard to prevent loading the entire wordlist + if len(partial_hint) > 0: tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) - async for r in self.execute_ffuf( - tempfile, root_url, prefix=f"{prefix}{delimeter}", suffix=f".{ext}" - ): - self.emit_event( - r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] + + if "shortname-directory" in self.shortname_to_event[hint].tags: + self.verbose( + f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix} and partial_hint: {partial_hint}" ) - async def finish(self): - if self.config.get("find_common_prefixes"): - per_host_collection = dict(self.per_host_collection) - self.per_host_collection.clear() - - for host, hint_tuple_list in per_host_collection.items(): - hint_list = [x[0] for x in hint_tuple_list] - - common_prefixes = find_common_prefixes(hint_list) - for prefix in common_prefixes: - self.verbose(f"Found common prefix: [{prefix}] for host [{host}]") - for hint_tuple in hint_tuple_list: - hint, url = hint_tuple - if hint.startswith(prefix): - partial_hint = hint[len(prefix) :] - - # safeguard to prevent loading the entire wordlist - if len(partial_hint) > 0: - tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) - - if "shortname-directory" in self.shortname_to_event[hint].tags: - self.verbose( - f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix} and partial_hint: {partial_hint}" + async for r in self.execute_ffuf(tempfile, url, prefix=prefix, exts=["/"]): + self.emit_event( + r["url"], + "URL_UNVERIFIED", + source=self.shortname_to_event[hint], + tags=[f"status-{r['status']}"], ) + elif "shortname-file" in self.shortname_to_event[hint].tags: + used_extensions = self.build_extension_list(self.shortname_to_event[hint]) - async for r in self.execute_ffuf(tempfile, url, prefix=prefix, exts=["/"]): + for ext in used_extensions: + self.verbose( + f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix}, extension: .{ext}, and partial_hint: {partial_hint}" + ) + async for r in self.execute_ffuf( + tempfile, url, prefix=prefix, suffix=f".{ext}" + ): self.emit_event( r["url"], "URL_UNVERIFIED", source=self.shortname_to_event[hint], tags=[f"status-{r['status']}"], ) - elif "shortname-file" in self.shortname_to_event[hint].tags: - used_extensions = self.build_extension_list(self.shortname_to_event[hint]) - - for ext in used_extensions: - self.verbose( - f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix}, extension: .{ext}, and partial_hint: {partial_hint}" - ) - async for r in self.execute_ffuf( - tempfile, url, prefix=prefix, suffix=f".{ext}" - ): - self.emit_event( - r["url"], - "URL_UNVERIFIED", - source=self.shortname_to_event[hint], - tags=[f"status-{r['status']}"], - ) diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index 8b61dbdd8..d5479c3a2 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -18,57 +18,57 @@ class filedownload(BaseModule): meta = {"description": "Download common filetypes such as PDF, DOCX, PPTX, etc."} options = { "extensions": [ - "bak", # Backup File - "bash", # Bash Script or Configuration - "bashrc", # Bash Script or Configuration - "conf", # Configuration File - "cfg", # Configuration File - "crt", # Certificate File - "csv", # Comma Separated Values File - "db", # SQLite Database File - "sqlite", # SQLite Database File - "doc", # Microsoft Word Document (Old Format) - "docx", # Microsoft Word Document - "exe", # Windows PE executable - "ica", # Citrix Independent Computing Architecture File - "indd", # Adobe InDesign Document - "ini", # Initialization File - "jar", # Java Archive - "key", # Private Key File - "pub", # Public Key File - "log", # Log File - "markdown", # Markdown File - "md", # Markdown File + "bak", # Backup File + "bash", # Bash Script or Configuration + "bashrc", # Bash Script or Configuration + "conf", # Configuration File + "cfg", # Configuration File + "crt", # Certificate File + "csv", # Comma Separated Values File + "db", # SQLite Database File + "sqlite", # SQLite Database File + "doc", # Microsoft Word Document (Old Format) + "docx", # Microsoft Word Document + "exe", # Windows PE executable + "ica", # Citrix Independent Computing Architecture File + "indd", # Adobe InDesign Document + "ini", # Initialization File + "jar", # Java Archive + "key", # Private Key File + "pub", # Public Key File + "log", # Log File + "markdown", # Markdown File + "md", # Markdown File "msi", # Windows setup file - "odg", # OpenDocument Graphics (LibreOffice, OpenOffice) - "odp", # OpenDocument Presentation (LibreOffice, OpenOffice) - "ods", # OpenDocument Spreadsheet (LibreOffice, OpenOffice) - "odt", # OpenDocument Text (LibreOffice, OpenOffice) - "pdf", # Adobe Portable Document Format - "pem", # Privacy Enhanced Mail (SSL certificate) - "png", # Portable Network Graphics Image - "pps", # Microsoft PowerPoint Slideshow (Old Format) - "ppsx", # Microsoft PowerPoint Slideshow - "ppt", # Microsoft PowerPoint Presentation (Old Format) - "pptx", # Microsoft PowerPoint Presentation - "ps1", # PowerShell Script - "raw", # Raw Image File Format - "rdp", # Remote Desktop Protocol File - "sh", # Shell Script - "sql", # SQL Database Dump - "swp", # Swap File (temporary file, often Vim) - "sxw", # OpenOffice.org Writer document - "tar", # Tar Archive + "odg", # OpenDocument Graphics (LibreOffice, OpenOffice) + "odp", # OpenDocument Presentation (LibreOffice, OpenOffice) + "ods", # OpenDocument Spreadsheet (LibreOffice, OpenOffice) + "odt", # OpenDocument Text (LibreOffice, OpenOffice) + "pdf", # Adobe Portable Document Format + "pem", # Privacy Enhanced Mail (SSL certificate) + "png", # Portable Network Graphics Image + "pps", # Microsoft PowerPoint Slideshow (Old Format) + "ppsx", # Microsoft PowerPoint Slideshow + "ppt", # Microsoft PowerPoint Presentation (Old Format) + "pptx", # Microsoft PowerPoint Presentation + "ps1", # PowerShell Script + "raw", # Raw Image File Format + "rdp", # Remote Desktop Protocol File + "sh", # Shell Script + "sql", # SQL Database Dump + "swp", # Swap File (temporary file, often Vim) + "sxw", # OpenOffice.org Writer document + "tar", # Tar Archive "tar.gz", # Gzip-Compressed Tar Archive - "zip", # Zip Archive - "txt", # Plain Text Document - "vbs", # Visual Basic Script - "wpd", # WordPerfect Document - "xls", # Microsoft Excel Spreadsheet (Old Format) - "xlsx", # Microsoft Excel Spreadsheet - "xml", # eXtensible Markup Language File - "yml", # YAML Ain't Markup Language - "yaml", # YAML Ain't Markup Language + "zip", # Zip Archive + "txt", # Plain Text Document + "vbs", # Visual Basic Script + "wpd", # WordPerfect Document + "xls", # Microsoft Excel Spreadsheet (Old Format) + "xlsx", # Microsoft Excel Spreadsheet + "xml", # eXtensible Markup Language File + "yml", # YAML Ain't Markup Language + "yaml", # YAML Ain't Markup Language ], "max_filesize": "10MB", } @@ -80,7 +80,9 @@ class filedownload(BaseModule): scope_distance_modifier = 1 async def setup(self): - self.extensions = list(set([e.lower().strip(".") for e in self.options.get("extensions", [])])) + self.extensions = list( + {e.lower().strip(".") for e in self.options.get("extensions", [])} + ) self.max_filesize = self.options.get("max_filesize", "10MB") self.download_dir = self.scan.home / "filedownload" self.helpers.mkdir(self.download_dir) @@ -116,8 +118,7 @@ async def handle_event(self, event): if any(url_lower.endswith(f".{e}") for e in self.extensions): await self.download_file(event.data) elif event.type == "HTTP_RESPONSE": - content_type = event.data["header"].get("content_type", "") - if content_type: + if content_type := event.data["header"].get("content_type", ""): url = event.data["url"] await self.download_file(url, content_type=content_type) @@ -137,18 +138,14 @@ def make_filename(self, url, content_type=None): url_path = parsed_url.path.strip("/") # try to get extension from URL path extension = Path(url_path).suffix.strip(".").lower() - if extension: - url_stem = url.rsplit(".", 1)[0] - else: - url_stem = str(url) + url_stem = url.rsplit(".", 1)[0] if extension else str(url) filename = f"{self.helpers.make_date()}_{self.helpers.tagify(url_stem)}" if not url_path: url_path = "unknown" filename = f"{filename}-{url_path}" # if that fails, try to get it from content type - if not extension: - if content_type and content_type in self.mime_db: - extension = self.mime_db[content_type] + if not extension and (content_type and content_type in self.mime_db): + extension = self.mime_db[content_type] if (not extension) or (extension not in self.extensions): self.debug(f'Extension "{extension}" at url "{url}" not in list of watched extensions.') diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 1485dc6b5..2e50f71c7 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -24,8 +24,7 @@ async def ping(self): async def request_url(self, query): url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains" - response = await self.request_with_fail_count(url, headers=self.headers) - return response + return await self.request_with_fail_count(url, headers=self.headers) def parse_results(self, r, query): return r.json().get("hosts", []) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index bf2c37097..525c30e94 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -1,7 +1,6 @@ from bbot.modules.base import BaseModule from bbot.core.errors import InteractshError - ssrf_params = [ "Dest", "Redirect", @@ -65,11 +64,7 @@ async def test(self, event): def process(self, event, r, subdomain_tag): response_token = self.parent_module.interactsh_domain.split(".")[0][::-1] - if response_token in r: - read_response = True - else: - read_response = False - + read_response = response_token in r self.parent_module.interactsh_subdomain_tags[subdomain_tag] = ( event, self.technique_description, @@ -86,14 +81,10 @@ def set_base_url(self, event): return event.data def create_paths(self): - query_string = "" - for param in ssrf_params: - query_string += f"{param}=http://SSRF_CANARY&" - - query_string_lower = "" - for param in ssrf_params: - query_string_lower += f"{param.lower()}=http://SSRF_CANARY&" - + query_string = "".join(f"{param}=http://SSRF_CANARY&" for param in ssrf_params) + query_string_lower = "".join( + f"{param.lower()}=http://SSRF_CANARY&" for param in ssrf_params + ) return [f"?{query_string.rstrip('&')}", f"?{query_string_lower.rstrip('&')}"] @@ -108,14 +99,14 @@ async def test(self, event): test_url = f"{event.data}" subdomain_tag = self.parent_module.helpers.rand_string(4, digits=False) - post_data = {} - for param in ssrf_params: - post_data[param] = f"http://{subdomain_tag}.{self.parent_module.interactsh_domain}" - + post_data = { + param: f"http://{subdomain_tag}.{self.parent_module.interactsh_domain}" + for param in ssrf_params + } subdomain_tag_lower = self.parent_module.helpers.rand_string(4, digits=False) post_data_lower = { k.lower(): f"http://{subdomain_tag_lower}.{self.parent_module.interactsh_domain}" - for k, v in post_data.items() + for k in post_data } post_data_list = [(subdomain_tag, post_data), (subdomain_tag_lower, post_data_lower)] @@ -189,8 +180,7 @@ async def handle_event(self, event): await s.test(event) def interactsh_callback(self, r): - full_id = r.get("full-id", None) - if full_id: + if full_id := r.get("full-id", None): if "." in full_id: match = self.interactsh_subdomain_tags.get(full_id.split(".")[0]) if not match: diff --git a/bbot/modules/git.py b/bbot/modules/git.py index dafe151d1..95ff2ed87 100644 --- a/bbot/modules/git.py +++ b/bbot/modules/git.py @@ -24,10 +24,7 @@ async def handle_event(self, event): tasks = [self.get_url(u) for u in urls] async for task in self.helpers.as_completed(tasks): result, url = await task - text = getattr(result, "text", "") - if not text: - text = "" - if text: + if text := getattr(result, "text", "") or "": if getattr(result, "status_code", 0) == 200 and "[core]" in text and not self.fp_regex.match(text): self.emit_event( {"host": str(event.host), "url": url, "description": f"Exposed .git config at {url}"}, diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 5f4c4a5e8..86d0bb52a 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -86,8 +86,7 @@ async def setup(self): self.proxy = self.scan.config.get("http_proxy", "") self.resolution_x = self.config.get("resolution_x") self.resolution_y = self.config.get("resolution_y") - output_path = self.config.get("output_path") - if output_path: + if output_path := self.config.get("output_path"): self.base_path = Path(output_path) / "gowitness" else: self.base_path = self.scan.home / "gowitness" @@ -99,7 +98,7 @@ async def setup(self): self.screenshot_path = self.base_path / "screenshots" self.command = self.construct_command() self.prepped = False - self.screenshots_taken = dict() + self.screenshots_taken = {} self.connections_logged = set() self.technologies_found = set() return True @@ -244,8 +243,8 @@ def cur_execute(self, cur, query): async def report(self): if self.screenshots_taken: self.success(f"{len(self.screenshots_taken):,} web screenshots captured. To view:") - self.success(f" - Start gowitness") + self.success(" - Start gowitness") self.success(f" - cd {self.base_path} && ./gowitness server") - self.success(f" - Browse to http://localhost:7171") + self.success(" - Browse to http://localhost:7171") else: - self.info(f"No web screenshots captured") + self.info("No web screenshots captured") diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py index d23f5c6cf..14dcd76c2 100644 --- a/bbot/modules/hackertarget.py +++ b/bbot/modules/hackertarget.py @@ -11,8 +11,7 @@ class hackertarget(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}" - response = await self.request_with_fail_count(url) - return response + return await self.request_with_fail_count(url) def parse_results(self, r, query): for line in r.text.splitlines(): diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py index c20f4c4b4..cd5fd013c 100644 --- a/bbot/modules/host_header.py +++ b/bbot/modules/host_header.py @@ -31,8 +31,7 @@ def rand_string(self, *args, **kwargs): return self.helpers.rand_string(*args, **kwargs) def interactsh_callback(self, r): - full_id = r.get("full-id", None) - if full_id: + if full_id := r.get("full-id", None): if "." in full_id: match = self.subdomain_tags.get(full_id.split(".")[0]) if match is None: @@ -132,7 +131,7 @@ async def handle_event(self, event): { "host": str(event.host), "url": event.data["url"], - "description": f"Duplicate Host Header Tolerated", + "description": "Duplicate Host Header Tolerated", }, "FINDING", event, diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 6341f16ea..615d851e4 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -100,14 +100,12 @@ async def handle_batch(self, *events): f"{self.max_response_size}", ] - dns_resolvers = ",".join(self.helpers.system_resolvers) - if dns_resolvers: + if dns_resolvers := ",".join(self.helpers.system_resolvers): command += ["-r", dns_resolvers] for hk, hv in self.scan.config.get("http_headers", {}).items(): command += ["-header", f"{hk}: {hv}"] - proxy = self.scan.config.get("http_proxy", "") - if proxy: + if proxy := self.scan.config.get("http_proxy", ""): command += ["-http-proxy", proxy] async for line in self.helpers.run_live(command, input=list(stdin), stderr=subprocess.DEVNULL): try: @@ -122,7 +120,7 @@ async def handle_batch(self, *events): self.debug(f'No HTTP status code for "{url}"') continue - source_event = stdin.get(j.get("input", ""), None) + source_event = stdin.get(j.get("input", "")) if source_event is None: self.warning(f"Unable to correlate source event from: {line}") @@ -130,24 +128,24 @@ async def handle_batch(self, *events): # discard 404s from unverified URLs path = j.get("path", "/") - if source_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/": + if ( + source_event.type == "URL_UNVERIFIED" + and status_code in {404} + and path != "/" + ): self.debug(f'Discarding 404 from "{url}"') continue # main URL tags = [f"status-{status_code}"] - httpx_ip = j.get("host", "") - if httpx_ip: + if httpx_ip := j.get("host", ""): tags.append(f"ip-{httpx_ip}") # detect login pages if is_login_page(j.get("body", "")): tags.append("login-page") - # grab title - title = self.helpers.tagify(j.get("title", ""), maxlen=30) - if title: + if title := self.helpers.tagify(j.get("title", ""), maxlen=30): tags.append(f"http-title-{title}") - url_event = self.make_event(url, "URL", source_event, tags=tags) - if url_event: + if url_event := self.make_event(url, "URL", source_event, tags=tags): if url_event != source_event: self.emit_event(url_event) else: diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index 7cc2e06dc..021379c5c 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -286,7 +286,6 @@ async def handle_event(self, event): if p.lower() in hunt_param_dict[k]: description = f"Found potential {k.upper()} parameter [{p}]" data = {"host": str(event.host), "description": description} - url = event.data.get("url", "") - if url: + if url := event.data.get("url", ""): data["url"] = url self.emit_event(data, "FINDING", event) diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 1e65c6e4c..f71304bc4 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -24,30 +24,27 @@ async def handle_event(self, event): email = entry.get("value", "") sources = entry.get("sources", []) if email: - email_event = self.make_event(email, "EMAIL_ADDRESS", event) - if email_event: + if email_event := self.make_event(email, "EMAIL_ADDRESS", event): self.emit_event(email_event) for source in sources: - domain = source.get("domain", "") - if domain: + if domain := source.get("domain", ""): self.emit_event(domain, "DNS_NAME", email_event) - url = source.get("uri", "") - if url: + if url := source.get("uri", ""): self.emit_event(url, "URL_UNVERIFIED", email_event) async def query(self, query): emails = [] url = ( - f"{self.base_url}/domain-search?domain={query}&api_key={self.api_key}" - + "&limit={page_size}&offset={offset}" + f"{self.base_url}/domain-search?domain={query}&api_key={self.api_key}" + + "&limit={page_size}&offset={offset}" ) agen = self.helpers.api_page_iter(url, page_size=self.limit) try: async for j in agen: - new_emails = j.get("data", {}).get("emails", []) - if not new_emails: + if new_emails := j.get("data", {}).get("emails", []): + emails += new_emails + else: break - emails += new_emails finally: agen.aclose() return emails diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index f9914e316..bea6d9f6d 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -39,7 +39,7 @@ async def detect(self, target): detections.append((method, test.status_code, technique)) elif ("Error Code0x80070002" in control.text) and ( - "Error Code0x00000000" in test.text + "Error Code0x00000000" in test.text ): detections.append((method, 0, technique)) technique = "HTTP Body Error Message" @@ -85,9 +85,8 @@ async def duplicate_check(self, target, method, url_hint, affirmative_status_cod if duplicate_check_results.status_code != affirmative_status_code: break - else: - duplicates.append(f"{base_hint}~{str(count)}") - count += 1 + duplicates.append(f"{base_hint}~{str(count)}") + count += 1 if count > 5: self.warning("Found more than 5 files with the same shortname. Will stop further duplicate checking.") @@ -97,13 +96,12 @@ async def duplicate_check(self, target, method, url_hint, affirmative_status_cod async def threaded_request(self, method, url, affirmative_status_code, c): r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) - if r is not None: - if r.status_code == affirmative_status_code: - return True, c + if r is not None and r.status_code == affirmative_status_code: + return True, c return None, c async def solve_shortname_recursive( - self, method, target, prefix, affirmative_status_code, extension_mode=False, node_count=0 + self, method, target, prefix, affirmative_status_code, extension_mode=False, node_count=0 ): url_hint_list = [] found_results = False @@ -122,7 +120,7 @@ async def solve_shortname_recursive( if result: found_results = True node_count += 1 - self.verbose(f"node_count: {str(node_count)} for node: {target}") + self.verbose(f"node_count: {node_count} for node: {target}") if node_count > self.config.get("max_node_count"): self.warning( f"iis_shortnames: max_node_count ({str(self.config.get('max_node_count'))}) exceeded for node: {target}. Affected branch will be terminated." @@ -134,9 +132,8 @@ async def solve_shortname_recursive( payload = encode_all(f"{prefix}{c}{wildcard}") url = f"{target}{payload}{suffix}" r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) - if r is not None: - if r.status_code == affirmative_status_code: - url_hint_list.append(f"{prefix}{c}") + if r is not None and r.status_code == affirmative_status_code: + url_hint_list.append(f"{prefix}{c}") url_hint_list += await self.solve_shortname_recursive( method, target, f"{prefix}{c}", affirmative_status_code, extension_mode, node_count=node_count @@ -152,8 +149,8 @@ async def handle_event(self, event): detections = await self.detect(normalized_url) - technique_strings = [] if detections: + technique_strings = [] for detection in detections: method, affirmative_status_code, technique = detection technique_strings.append(f"{method} ({technique})") @@ -175,7 +172,7 @@ async def handle_event(self, event): file_name_hints = list( set(await self.solve_shortname_recursive(method, normalized_url, "", affirmative_status_code)) ) - if len(file_name_hints) == 0: + if not file_name_hints: continue else: valid_method_confirmed = True @@ -207,15 +204,10 @@ async def handle_event(self, event): url_hint_list.append(z) for url_hint in url_hint_list: - if "." in url_hint: - hint_type = "shortname-file" - else: - hint_type = "shortname-directory" + hint_type = "shortname-file" if "." in url_hint else "shortname-directory" self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type]) async def filter_event(self, event): if "dir" in event.tags: - if self.normalize_url(event.data) not in self.scanned_tracker: - return True - return False + return self.normalize_url(event.data) not in self.scanned_tracker return False diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 95a5a848f..2cbd44aea 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -41,8 +41,7 @@ class CSPExtractor(BaseExtractor): def extract_domains(self, csp): domains = dns_name_regex.findall(csp) - unique_domains = set(domains) - return unique_domains + return set(domains) async def search(self, content, event, **kwargs): async for csp, name in self._search(content, event, **kwargs): @@ -59,7 +58,7 @@ class HostnameExtractor(BaseExtractor): def __init__(self, excavate): for i, r in enumerate(excavate.scan.dns_regexes): - self.regexes[f"dns_name_{i+1}"] = r.pattern + self.regexes[f"dns_name_{i + 1}"] = r.pattern super().__init__(excavate) def report(self, result, name, event, **kwargs): @@ -98,13 +97,13 @@ async def search(self, content, event, **kwargs): url_in_scope = self.excavate.scan.in_scope(url_event) is_spider_danger = self.excavate.helpers.is_spider_danger(event, result) if ( - ( - urls_found >= self.web_spider_links_per_page and url_in_scope - ) # if we exceeded the max number of links - or (consider_spider_danger and is_spider_danger) # or if there's spider danger - or ( + ( + urls_found >= self.web_spider_links_per_page and url_in_scope + ) # if we exceeded the max number of links + or (consider_spider_danger and is_spider_danger) # or if there's spider danger + or ( (not consider_spider_danger) and (web_spider_distance > self.excavate.max_redirects) - ) # or if the spider distance is way out of control (greater than max_redirects) + ) # or if the spider distance is way out of control (greater than max_redirects) ): url_event.add_tag("spider-danger") @@ -148,10 +147,9 @@ def report(self, result, name, event, **kwargs): parsed_uri = self.excavate.helpers.urlparse(result) host, port = self.excavate.helpers.split_host_port(parsed_uri.netloc) # Handle non-HTTP URIs (ftp, s3, etc.) - if not "http" in parsed_uri.scheme.lower(): + if "http" not in parsed_uri.scheme.lower(): event_data = {"host": str(host), "description": f"Non-HTTP URI: {result}"} - parsed_url = getattr(event, "parsed", None) - if parsed_url: + if parsed_url := getattr(event, "parsed", None): event_data["url"] = parsed_url.geturl() self.excavate.emit_event( event_data, @@ -216,9 +214,7 @@ def report(self, result, name, event, **kwargs): try: j.decode(result, options={"verify_signature": False}) jwt_headers = j.get_unverified_header(result) - tags = [] - if jwt_headers["alg"].upper()[0:2] == "HS": - tags = ["crackable"] + tags = ["crackable"] if jwt_headers["alg"].upper()[:2] == "HS" else [] description = f"JWT Identified [{result}]" self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url", ""), "description": description}, diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 4aa3bb616..8604eb737 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -35,8 +35,12 @@ class speculate(BaseInternalModule): _priority = 4 async def setup(self): - self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()]) - self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()]) + self.open_port_consumers = any( + "OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values() + ) + self.portscanner_enabled = any( + "portscan" in m.flags for m in self.scan.modules.values() + ) self.range_to_ip = True self.dns_resolution = self.scan.config.get("dns_resolution", True) @@ -57,7 +61,9 @@ async def setup(self): self.hugewarning( f"Selected target ({target_len:,} hosts) is too large, skipping IP_RANGE --> IP_ADDRESS speculation" ) - self.hugewarning(f"Enabling a port scanner (nmap or masscan) module is highly recommended") + self.hugewarning( + "Enabling a port scanner (nmap or masscan) module is highly recommended" + ) self.range_to_ip = False return True @@ -80,15 +86,14 @@ async def handle_event(self, event): # generate open ports emit_open_ports = self.open_port_consumers and not self.portscanner_enabled # from URLs - if event.type == "URL" or (event.type == "URL_UNVERIFIED" and emit_open_ports): - if event.host and event.port not in self.ports: - self.emit_event( - self.helpers.make_netloc(event.host, event.port), - "OPEN_TCP_PORT", - source=event, - internal=True, - quick=(event.type == "URL"), - ) + if (event.type == "URL" or (event.type == "URL_UNVERIFIED" and emit_open_ports)) and (event.host and event.port not in self.ports): + self.emit_event( + self.helpers.make_netloc(event.host, event.port), + "OPEN_TCP_PORT", + source=event, + internal=True, + quick=(event.type == "URL"), + ) # generate sub-directory URLS from URLS if event.type == "URL": @@ -103,12 +108,10 @@ async def handle_event(self, event): # from hosts if emit_open_ports: - # don't act on unresolved DNS_NAMEs - usable_dns = False - if event.type == "DNS_NAME": - if (not self.dns_resolution) or ("a-record" in event.tags or "aaaa-record" in event.tags): - usable_dns = True - + usable_dns = event.type == "DNS_NAME" and ( + (not self.dns_resolution) + or ("a-record" in event.tags or "aaaa-record" in event.tags) + ) if event.type == "IP_ADDRESS" or usable_dns: for port in self.ports: self.emit_event( @@ -124,10 +127,12 @@ async def handle_event(self, event): async def filter_event(self, event): # don't accept IP_RANGE --> IP_ADDRESS events from self - if str(event.module) == "speculate": - if not (event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE"): - return False - # don't accept errored DNS_NAMEs - if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")): + if str(event.module) == "speculate" and ( + event.type != "IP_ADDRESS" + or str(getattr(event.source, "type")) != "IP_RANGE" + ): return False - return True + # don't accept errored DNS_NAMEs + return all( + t not in event.tags for t in ("unresolved", "a-error", "aaaa-error") + ) diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 4a203e55b..dcdd1f28f 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -52,10 +52,8 @@ async def handle_event(self, event): self.verbose(f"Error retrieving results for {event.data}", trace=True) return - geo_data = {k: v for k, v in geo_data.items() if v is not None} - if geo_data: + if geo_data := {k: v for k, v in geo_data.items() if v is not None}: self.emit_event(geo_data, "GEOLOCATION", event) elif "error" in geo_data: - error_msg = geo_data.get("error").get("error_message", "") - if error_msg: + if error_msg := geo_data.get("error").get("error_message", ""): self.warning(error_msg) diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index 1ce8cf0f9..f010748c3 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -19,16 +19,14 @@ async def setup(self): return True async def filter_event(self, event): - if str(event.module) in ("speculate", "ipneighbor"): - return False - return True + return str(event.module) not in {"speculate", "ipneighbor"} async def handle_event(self, event): main_ip = event.host netmask = main_ip.max_prefixlen - min(main_ip.max_prefixlen, self.num_bits) network = ipaddress.ip_network(f"{main_ip}/{netmask}", strict=False) subnet_hash = hash(network) - if not subnet_hash in self.processed: + if subnet_hash not in self.processed: self.processed.add(subnet_hash) for ip in network: if ip != main_ip: diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index 031ac272c..b1cb56bca 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -42,10 +42,8 @@ async def handle_event(self, event): except Exception: self.verbose(f"Error retrieving results for {event.data}", trace=True) return - geo_data = {k: v for k, v in geo_data.items() if v is not None} - if geo_data: + if geo_data := {k: v for k, v in geo_data.items() if v is not None}: self.emit_event(geo_data, "GEOLOCATION", event) elif "error" in geo_data: - error_msg = geo_data.get("error").get("info", "") - if error_msg: + if error_msg := geo_data.get("error").get("info", ""): self.warning(error_msg) diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 45053755a..b0170e4ea 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -29,13 +29,10 @@ async def ping(self): async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" - response = await self.request_with_fail_count(url, headers=self.headers) - return response + return await self.request_with_fail_count(url, headers=self.headers) def parse_results(self, r, query=None): - json = r.json() - if json: + if json := r.json(): for entry in json: - subdomain = entry.get("subdomain", "") - if subdomain: + if subdomain := entry.get("subdomain", ""): yield subdomain diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index 647a27a3d..37681b6ac 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -70,8 +70,8 @@ async def setup(self): self.run_time = self.helpers.make_date() self.use_cache = self.config.get("use_cache", False) - self.ping_cache = self.scan.home / f"masscan_ping.txt" - self.syn_cache = self.scan.home / f"masscan_syn.txt" + self.ping_cache = self.scan.home / "masscan_ping.txt" + self.syn_cache = self.scan.home / "masscan_syn.txt" if self.use_cache: files_exist = self.ping_cache.is_file() or self.syn_cache.is_file() files_empty = self.helpers.filesize(self.ping_cache) == 0 and self.helpers.filesize(self.syn_cache) == 0 @@ -211,7 +211,7 @@ def emit_from_cache(self): if cached_pings: self.success(f"{len(cached_pings):,} hosts loaded from previous ping scan") else: - self.verbose(f"No hosts cached from previous ping scan") + self.verbose("No hosts cached from previous ping scan") for ip in cached_pings: if self.scan.stopping: break @@ -224,7 +224,7 @@ def emit_from_cache(self): if cached_syns: self.success(f"{len(cached_syns):,} hosts loaded from previous SYN scan") else: - self.warning(f"No hosts cached from previous SYN scan") + self.warning("No hosts cached from previous SYN scan") for line in cached_syns: if self.scan.stopping: break diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 5dcd10ded..be2ad100c 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -114,20 +114,18 @@ async def handle_event(self, event): self.emit_result(hostname, event, query) def abort_if(self, event): - if not event.scope_distance == 0: + if event.scope_distance != 0: return True, "event is not in scope" if "wildcard" in event.tags: return True, "event is a wildcard" def emit_result(self, result, source_event, query): - if not result == source_event: + if result != source_event: kwargs = {"abort_if": self.abort_if} self.emit_event(result, "DNS_NAME", source_event, **kwargs) def already_processed(self, hostname): - if hash(hostname) in self.processed: - return True - return False + return hash(hostname) in self.processed async def massdns(self, domain, subdomains): subdomains = list(subdomains) @@ -151,7 +149,7 @@ async def massdns(self, domain, subdomains): abort_msg = f"Aborting massdns on {domain} due to false positive" canary_result = await self._canary_check(domain) if canary_result: - self.info(abort_msg + f": {canary_result}") + self.info(f"{abort_msg}: {canary_result}") return [] else: self.log.trace(f"Canary result for {domain}: {canary_result}") @@ -176,7 +174,7 @@ async def massdns(self, domain, subdomains): if len(results) > 50: canary_result = await self._canary_check(domain) if canary_result: - self.info(abort_msg + f": {canary_result}") + self.info(f"{abort_msg}: {canary_result}") return [] else: self.log.trace(f"Canary result for {domain}: {canary_result}") @@ -278,7 +276,7 @@ async def _massdns(self, domain, subdomains): rdtype = answer.get("type", "").upper() # avoid garbage answers like this: # 8AAAA queries have been locally blocked by dnscrypt-proxy/Set block_ipv6 to false to disable this feature - if data and rdtype and not " " in data: + if data and rdtype and " " not in data: hostname_hash = hash(hostname) if hostname_hash not in hosts_yielded: hosts_yielded.add(hostname_hash) @@ -289,7 +287,9 @@ async def finish(self): # if we have a lot of rounds to make, don't try mutations on less-populated domains trimmed_found = [] if found: - avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50]) + avg_subdomains = sum( + len(subdomains) for domain, subdomains in found[:50] + ) / len(found[:50]) for i, (domain, subdomains) in enumerate(found): # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)): @@ -343,13 +343,13 @@ def add_mutation(_domain_hash, m): continue add_mutation(domain_hash, first_segment) for word in self.helpers.extract_words( - first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes + first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes ): add_mutation(domain_hash, word) # numbers + devops mutations for mutation in self.helpers.word_cloud.mutations( - subdomains, cloud=False, numbers=3, number_padding=1 + subdomains, cloud=False, numbers=3, number_padding=1 ): for delimiter in ("", ".", "-"): m = delimiter.join(mutation).lower() @@ -357,12 +357,12 @@ def add_mutation(_domain_hash, m): # special dns mutator for subdomain in self.helpers.word_cloud.dns_mutator.mutations( - subdomains, max_mutations=self.max_mutations + subdomains, max_mutations=self.max_mutations ): add_mutation(domain_hash, subdomain) if mutations: - self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})") + self.info(f"Trying {len(mutations):,} mutations against {domain} ({i + 1}/{len(found)})") results = list(await self.massdns(query, mutations)) for hostname in results: source_event = self.source_events.get(hostname) @@ -387,22 +387,20 @@ def add_found(self, host): try: self.found[domain].add(subdomain) except KeyError: - self.found[domain] = set((subdomain,)) + self.found[domain] = {subdomain} async def gen_subdomains(self, prefixes, domain): for p in prefixes: - d = f"{p}.{domain}" - yield d + yield f"{p}.{domain}" def gen_random_subdomains(self, n=50): delimeters = (".", "-") lengths = list(range(3, 8)) - for i in range(0, max(0, n - 5)): + for i in range(max(0, n - 5)): d = delimeters[i % len(delimeters)] l = lengths[i % len(lengths)] - segments = list(random.choice(self.devops_mutations) for _ in range(l)) + segments = [random.choice(self.devops_mutations) for _ in range(l)] segments.append(self.helpers.rand_string(length=8, digits=False)) - subdomain = d.join(segments) - yield subdomain + yield d.join(segments) for _ in range(5): yield self.helpers.rand_string(length=8, digits=False) diff --git a/bbot/modules/myssl.py b/bbot/modules/myssl.py index a08c885ed..dabcfbfa9 100644 --- a/bbot/modules/myssl.py +++ b/bbot/modules/myssl.py @@ -19,7 +19,6 @@ def parse_results(self, r, query): if json and isinstance(json, dict): data = json.get("data", []) for d in data: - hostname = d.get("domain", "").lower() - if hostname: + if hostname := d.get("domain", "").lower(): results.add(hostname) return results diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py index 5e485e5ac..ff69632cb 100644 --- a/bbot/modules/nmap.py +++ b/bbot/modules/nmap.py @@ -36,7 +36,7 @@ async def setup(self): async def handle_batch(self, *events): target = self.helpers.make_target(*events) - targets = list(set(str(e.data) for e in events)) + targets = list({str(e.data) for e in events}) command, output_file = self.construct_command(targets) try: await self.helpers.run(command, sudo=True) @@ -73,10 +73,7 @@ def construct_command(self, targets): ] if self.skip_host_discovery: command += ["-Pn"] - if ports: - command += ["-p", ports] - else: - command += ["--top-ports", top_ports] + command += ["-p", ports] if ports else ["--top-ports", top_ports] command += targets return command, temp_filename @@ -104,11 +101,11 @@ def __init__(self, xml): self.hostnames = [] for hostname in self.etree.findall("hostnames/hostname"): hostname = hostname.attrib.get("name") - if hostname and not hostname in self.hostnames: + if hostname and hostname not in self.hostnames: self.hostnames.append(hostname) # convenient port information - self.scripts = dict() + self.scripts = {} self.open_ports = [] self.closed_ports = [] self.filtered_ports = [] diff --git a/bbot/modules/nsec.py b/bbot/modules/nsec.py index cafed732e..ccd26912b 100644 --- a/bbot/modules/nsec.py +++ b/bbot/modules/nsec.py @@ -9,9 +9,7 @@ class NSEC(BaseModule): _max_event_handlers = 5 async def filter_event(self, event): - if "ns-record" in event.tags: - return True - return False + return "ns-record" in event.tags async def handle_event(self, event): emitted_finding = False diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 76e93c595..a02757eee 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -96,30 +96,29 @@ async def handle_event(self, event): "FINDING", source=event, ) - fqdn = result.get("FQDN", "") - if fqdn: + if fqdn := result.get("FQDN", ""): self.emit_event(fqdn, "DNS_NAME", source=event) break async def filter_event(self, event): if self.try_all: return True - if event.type == "HTTP_RESPONSE": - if "www-authenticate" in event.data["header-dict"]: - header_value = event.data["header-dict"]["www-authenticate"].lower() - if "ntlm" in header_value or "negotiate" in header_value: - return True + if event.type == "HTTP_RESPONSE" and "www-authenticate" in event.data["header-dict"]: + header_value = event.data["header-dict"]["www-authenticate"].lower() + if "ntlm" in header_value or "negotiate" in header_value: + return True return False async def handle_url(self, event): - if event.type == "URL": - urls = { + urls = ( + { event.data, } - else: - urls = { + if event.type == "URL" + else { event.data["url"], } + ) if self.try_all: for endpoint in ntlm_discovery_endpoints: urls.add(f"{event.parsed.scheme}://{event.parsed.netloc}/{endpoint}") @@ -138,12 +137,12 @@ async def check_ntlm(self, test_url): # use lower timeout value http_timeout = self.config.get("httpx_timeout", 5) r = await self.helpers.request(test_url, headers=NTLM_test_header, allow_redirects=False, timeout=http_timeout) - ntlm_resp = r.headers.get("WWW-Authenticate", "") - if ntlm_resp: + if ntlm_resp := r.headers.get("WWW-Authenticate", ""): ntlm_resp_b64 = max(ntlm_resp.split(","), key=lambda x: len(x)).split()[-1] try: - ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64) - if ntlm_resp_decoded: + if ntlm_resp_decoded := self.helpers.ntlm.ntlmdecode( + ntlm_resp_b64 + ): return ntlm_resp_decoded, test_url except NTLMError as e: self.verbose(str(e)) diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index f4d592511..3b3c21719 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -41,18 +41,15 @@ async def handle_event(self, event): self.processed.add(domain_hash) oidc_tasks.append(self.helpers.create_task(self.getoidc(f"https://login.windows.net/{domain}"))) - if event.type == "URL_UNVERIFIED": - url = event.data - else: - url = f"https://{event.data}" - + url = event.data if event.type == "URL_UNVERIFIED" else f"https://{event.data}" oauth_tasks = [] if self.try_all or any(t in event.tags for t in ("oauth-token-endpoint",)): oauth_tasks.append(self.helpers.create_task(self.getoauth(url))) if self.try_all or any(t in event.tags for t in ("ms-auth-url",)): - for u in self.url_and_base(url): - oidc_tasks.append(self.helpers.create_task(self.getoidc(u))) - + oidc_tasks.extend( + self.helpers.create_task(self.getoidc(u)) + for u in self.url_and_base(url) + ) for oidc_task in oidc_tasks: url, token_endpoint, oidc_results = await oidc_task if token_endpoint: @@ -103,7 +100,7 @@ async def getoidc(self, url): results = set() if not url.endswith("openid-configuration"): url = url.strip("/") + "/.well-known/openid-configuration" - url_hash = hash("OIDC:" + url) + url_hash = hash(f"OIDC:{url}") token_endpoint = "" if url_hash not in self.processed: self.processed.add(url_hash) @@ -122,20 +119,18 @@ async def getoidc(self, url): return url, token_endpoint, results async def getoauth(self, url): - data = { - "grant_type": "authorization_code", - "client_id": "xxx", - "redirect_uri": "https://example.com", - "code": "xxx", - "client_secret": "xxx", - } - url_hash = hash("OAUTH:" + url) + url_hash = hash(f"OAUTH:{url}") if url_hash not in self.processed: self.processed.add(url_hash) + data = { + "grant_type": "authorization_code", + "client_id": "xxx", + "redirect_uri": "https://example.com", + "code": "xxx", + "client_secret": "xxx", + } r = await self.helpers.request(url, method="POST", data=data) if r is None: return - if r.status_code in (400, 401): - if "json" in r.headers.get("content-type", "").lower(): - if any(x in r.text.lower() for x in ("invalid_grant", "invalid_client")): - return url + if "json" in r.headers.get("content-type", "").lower() and any(x in r.text.lower() for x in ("invalid_grant", "invalid_client")) and r.status_code in (400, 401): + return url diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py index 72f2e1d5b..2b3dd6766 100644 --- a/bbot/modules/otx.py +++ b/bbot/modules/otx.py @@ -17,6 +17,5 @@ def parse_results(self, r, query): j = r.json() if isinstance(j, dict): for entry in j.get("passive_dns", []): - subdomain = entry.get("hostname", "") - if subdomain: + if subdomain := entry.get("hostname", ""): yield subdomain diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index 56e94aaa2..3a15d0a46 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -37,14 +37,13 @@ class asset_inventory(CSV): async def setup(self): self.assets = {} self.open_port_producers = "httpx" in self.scan.modules or any( - ["portscan" in m.flags for m in self.scan.modules.values()] + "portscan" in m.flags for m in self.scan.modules.values() ) self.use_previous = self.config.get("use_previous", False) self.summary_netmask = self.config.get("summary_netmask", 16) self.emitted_contents = False self._ran_hooks = False - ret = await super().setup() - return ret + return await super().setup() async def filter_event(self, event): if event._internal: @@ -73,7 +72,7 @@ def increment_stat(stat, value): totals[stat] += 1 except KeyError: totals[stat] = 1 - if not stat in stats: + if stat not in stats: stats[stat] = {} try: stats[stat][value] += 1 @@ -122,7 +121,7 @@ def sort_key(asset): stats_sorted = sorted(stats[header].items(), key=lambda x: x[-1], reverse=True) total = totals[header] for k, v in stats_sorted: - table.append([str(k), f"{v:,}/{total} ({v/total*100:.1f}%)"]) + table.append([str(k), f"{v:,}/{total} ({v / total * 100:.1f}%)"]) self.log_table(table, table_header, table_name=f"asset-inventory-{header}") if self._file is not None: @@ -204,20 +203,18 @@ def absorb_csv_row(self, row): self.ip_addresses = set(_make_ip_list(row.get("IP(s)", ""))) # ports ports = [i.strip() for i in row.get("Open Ports", "").split(",")] - self.ports.update(set(i for i in ports if i and is_port(i))) + self.ports.update({i for i in ports if i and is_port(i)}) # findings findings = [i.strip() for i in row.get("Findings", "").splitlines()] - self.findings.update(set(i for i in findings if i)) + self.findings.update({i for i in findings if i}) # technologies technologies = [i.strip() for i in row.get("Description", "").splitlines()] - self.technologies.update(set(i for i in technologies if i)) + self.technologies.update({i for i in technologies if i}) # risk rating risk_rating = row.get("Risk Rating", "").strip() if risk_rating and risk_rating.isdigit() and int(risk_rating) > self.risk_rating: self.risk_rating = int(risk_rating) - # provider - provider = row.get("Provider", "").strip() - if provider: + if provider := row.get("Provider", "").strip(): self.provider = provider # custom fields for k, v in row.items(): @@ -237,13 +234,11 @@ def absorb_event(self, event): self.ports.add(str(event.port)) if event.type == "FINDING": - location = event.data.get("url", event.data.get("host", "")) - if location: + if location := event.data.get("url", event.data.get("host", "")): self.findings.add(f"{location}:{event.data['description']}") if event.type == "VULNERABILITY": - location = event.data.get("url", event.data.get("host", "")) - if location: + if location := event.data.get("url", event.data.get("host", "")): self.findings.add(f"{location}:{event.data['description']}:{event.data['severity']}") severity_int = severity_map.get(event.data.get("severity", "N/A"), 0) if severity_int > self.risk_rating: @@ -269,8 +264,7 @@ def _make_hostkey(host, ips): If they're private, we dedupe by the IPs themselves """ ips = _make_ip_list(ips) - is_private = ips and all(is_ip(i) and i.is_private for i in ips) - if is_private: + if is_private := ips and all(is_ip(i) and i.is_private for i in ips): return ",".join(sorted([str(i) for i in ips])) return str(host) diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 204a8f44a..20f260c44 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -14,13 +14,12 @@ def _event_precheck(self, event): if event.type in ("FINISHED",): return True, "its type is FINISHED" if self.errored: - return False, f"module is in error state" + return False, "module is in error state" # exclude non-watched types - if not any(t in self.get_watched_events() for t in ("*", event.type)): + if all(t not in self.get_watched_events() for t in ("*", event.type)): return False, "its type is not in watched_events" - if self.target_only: - if "target" not in event.tags: - return False, "it did not meet target_only filter criteria" + if self.target_only and "target" not in event.tags: + return False, "it did not meet target_only filter criteria" # exclude certain URLs (e.g. javascript): if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: return False, "its extension was listed in url_extension_httpx_only" @@ -41,16 +40,12 @@ def _event_precheck(self, event): # if event is an IP address that was speculated from a CIDR source_is_range = getattr(event.source, "type", "") == "IP_RANGE" if ( - source_is_range - and event.type == "IP_ADDRESS" - and str(event.module) == "speculate" - and self.name != "speculate" - ): - # and the current module listens for both ranges and CIDRs - if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]): - # then skip the event. - # this helps avoid double-portscanning both an individual IP and its parent CIDR. - return False, "module consumes IP ranges directly" + source_is_range + and event.type == "IP_ADDRESS" + and str(event.module) == "speculate" + and self.name != "speculate" + ) and all(x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")): + return False, "module consumes IP ranges directly" return True, "precheck succeeded" def _prep_output_dir(self, filename): diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py index 3acb68f0b..95d5071bc 100644 --- a/bbot/modules/output/discord.py +++ b/bbot/modules/output/discord.py @@ -21,9 +21,9 @@ async def setup(self): self.webhook_url = self.config.get("webhook_url", "") self.min_severity = self.config.get("min_severity", "LOW").strip().upper() assert ( - self.min_severity in self.vuln_severities + self.min_severity in self.vuln_severities ), f"min_severity must be one of the following: {','.join(self.vuln_severities)}" - self.allowed_severities = self.vuln_severities[self.vuln_severities.index(self.min_severity) :] + self.allowed_severities = self.vuln_severities[self.vuln_severities.index(self.min_severity):] if not self.webhook_url: self.warning("Must set Webhook URL") return False @@ -40,16 +40,15 @@ async def handle_event(self, event): status_code = getattr(response, "status_code", 0) if self.evaluate_response(response): break - else: - response_data = getattr(response, "text", "") - try: - retry_after = response.json().get("retry_after", 1) - except Exception: - retry_after = 1 - self.verbose( - f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds" - ) - await self.helpers.sleep(retry_after) + response_data = getattr(response, "text", "") + try: + retry_after = response.json().get("retry_after", 1) + except Exception: + retry_after = 1 + self.verbose( + f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds" + ) + await self.helpers.sleep(retry_after) def get_watched_events(self): if self._watched_events is None: @@ -62,7 +61,7 @@ def get_watched_events(self): async def filter_event(self, event): if event.type == "VULNERABILITY": severity = event.data.get("severity", "UNKNOWN") - if not severity in self.allowed_severities: + if severity not in self.allowed_severities: return False, f"{severity} is below min_severity threshold" return True @@ -79,11 +78,10 @@ def format_message_other(self, event): return f"""**`{event_type}`**\n```yaml\n{event_yaml}```""" def get_severity_color(self, event): - if event.type == "VULNERABILITY": - severity = event.data.get("severity", "UNKNOWN") - return f"{event.type} ({severity})", event.severity_colors[severity] - else: + if event.type != "VULNERABILITY": return event.type, "🟦" + severity = event.data.get("severity", "UNKNOWN") + return f"{event.type} ({severity})", event.severity_colors[severity] def format_message(self, event): if isinstance(event.data, str): diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 10ca1c8df..e73b2455f 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -28,8 +28,7 @@ async def setup(self): self.method = self.config.get("method", "POST") self.timeout = self.config.get("timeout", 10) self.headers = {} - bearer = self.config.get("bearer", "") - if bearer: + if bearer := self.config.get("bearer", ""): self.headers["Authorization"] = f"Bearer {bearer}" username = self.config.get("username", "") password = self.config.get("password", "") diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index 793f26c32..35e25c8ee 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -41,7 +41,7 @@ async def handle_event(self, event): source_chain = [] current_parent = event.source - while not current_parent.type == "SCAN": + while current_parent.type != "SCAN": source_chain.append( f" ({current_parent.module})---> [{current_parent.type}]:{html.escape(current_parent.pretty_string)}" ) @@ -49,9 +49,9 @@ async def handle_event(self, event): source_chain.reverse() source_chain_text = ( - "".join(source_chain) - + f" ({event.module})---> " - + f"[{event.type}]:{html.escape(event.pretty_string)}" + "".join(source_chain) + + f" ({event.module})---> " + + f"[{event.type}]:{html.escape(event.pretty_string)}" ) self.web_assets[host]["URL"].append(f"**{html.escape(event.data)}**: {source_chain_text}") diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index d4d6d9966..48a5cc0b9 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -27,7 +27,7 @@ async def ws(self, rebuild=False): if self._ws is None or rebuild: kwargs = {"close_timeout": 0.5} if self.token: - kwargs.update({"extra_headers": {"Authorization": f"Bearer {self.token}"}}) + kwargs["extra_headers"] = {"Authorization": f"Bearer {self.token}"} verbs = ("Building", "Built") if rebuild: verbs = ("Rebuilding", "Rebuilt") diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index 251931f58..1871632a8 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -36,9 +36,7 @@ async def check_batch(self, compare_helper, url, cookie_list): def gen_count_args(self, url): cookie_count = 40 - while 1: - if cookie_count < 0: - break - fake_cookies = {self.rand_string(14): self.rand_string(14) for _ in range(0, cookie_count)} + while 1 and cookie_count >= 0: + fake_cookies = {self.rand_string(14): self.rand_string(14) for _ in range(cookie_count)} yield cookie_count, (url,), {"cookies": fake_cookies} cookie_count -= 5 diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 7891e05cf..4740965fe 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -34,9 +34,7 @@ async def check_batch(self, compare_helper, url, getparam_list): def gen_count_args(self, url): getparam_count = 40 - while 1: - if getparam_count < 0: - break - fake_getparams = {self.rand_string(14): self.rand_string(14) for _ in range(0, getparam_count)} + while 1 and getparam_count >= 0: + fake_getparams = {self.rand_string(14): self.rand_string(14) for _ in range(getparam_count)} yield getparam_count, (self.helpers.add_get_params(url, fake_getparams).geturl(),), {} getparam_count -= 5 diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 65880d9c8..ac8b050f9 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -1,3 +1,4 @@ +import contextlib from bbot.modules.base import BaseModule from bbot.core.errors import HttpCompareError from bbot.core.helpers.misc import extract_params_json, extract_params_xml, extract_params_html @@ -77,15 +78,13 @@ class paramminer_headers(BaseModule): async def setup(self): self.event_dict = {} self.already_checked = set() - wordlist = self.config.get("wordlist", "") - if not wordlist: - wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}" + wordlist = self.config.get("wordlist", "") or f"{self.helpers.wordlist_dir}/{self.default_wordlist}" self.debug(f"Using wordlist: [{wordlist}]") - self.wl = set( + self.wl = { h.strip().lower() for h in self.helpers.read_file(await self.helpers.wordlist(wordlist)) if len(h) > 0 and "%" not in h - ) + } # check against the boring list (if the option is set) @@ -105,7 +104,7 @@ async def do_mining(self, wl, url, batch_size, compare_helper): results = set() abort_threshold = 15 - try: + with contextlib.suppress(AssertionError): for group in self.helpers.grouper(wl, batch_size): async for result, reasons, reflection in self.binary_search(compare_helper, url, group): results.add((result, ",".join(reasons), reflection)) @@ -115,8 +114,6 @@ async def do_mining(self, wl, url, batch_size, compare_helper): ) results.clear() assert False - except AssertionError: - pass return results def process_results(self, event, results): @@ -142,7 +139,7 @@ async def handle_event(self, event): self.debug(f"Error initializing compare helper: {e}") return batch_size = await self.count_test(url) - if batch_size == None or batch_size <= 0: + if batch_size is None or batch_size <= 0: self.debug(f"Failed to get baseline max {self.compare_mode} count, aborting") return self.debug(f"Resolved batch_size at {str(batch_size)}") @@ -158,9 +155,10 @@ async def handle_event(self, event): wl = set(self.wl) if self.config.get("http_extract"): - extracted_words = self.load_extracted_words(event.data.get("body"), event.data.get("content_type")) - if extracted_words: - self.debug(f"Extracted {str(len(extracted_words))} words from {url}") + if extracted_words := self.load_extracted_words( + event.data.get("body"), event.data.get("content_type") + ): + self.debug(f"Extracted {len(extracted_words)} words from {url}") self.extracted_words_master.update(extracted_words - wl) wl |= extracted_words @@ -181,17 +179,16 @@ async def count_test(self, url): return for count, args, kwargs in self.gen_count_args(url): r = await self.helpers.request(*args, **kwargs) - if r is not None and not ((str(r.status_code)[0] in ("4", "5"))): + if r is not None and str(r.status_code)[0] not in ("4", "5"): return count def gen_count_args(self, url): header_count = 95 - while 1: - if header_count < 0: - break - fake_headers = {} - for i in range(0, header_count): - fake_headers[self.rand_string(14)] = self.rand_string(14) + while 1 and header_count >= 0: + fake_headers = { + self.rand_string(14): self.rand_string(14) + for _ in range(header_count) + } yield header_count, (url,), {"headers": fake_headers} header_count -= 5 @@ -218,13 +215,11 @@ async def binary_search(self, compare_helper, url, group, reasons=None, reflecti async for r in self.binary_search(compare_helper, url, group_slice, reasons, reflection): yield r else: - self.warning(f"Submitted group of size 0 to binary_search()") + self.warning("Submitted group of size 0 to binary_search()") async def check_batch(self, compare_helper, url, header_list): rand = self.rand_string() - test_headers = {} - for header in header_list: - test_headers[header] = rand + test_headers = {header: rand for header in header_list} return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1)) async def finish(self): diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index c1e0773c3..ec9ddc6bd 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -19,7 +19,7 @@ async def handle_event(self, event): results = await self.query(query) if results: for hostname in results: - if not hostname == event: + if hostname != event: self.emit_event(hostname, "EMAIL_ADDRESS", event, abort_if=self.abort_if) async def query(self, query): diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py index 088288ddb..78cec3147 100644 --- a/bbot/modules/rapiddns.py +++ b/bbot/modules/rapiddns.py @@ -11,8 +11,7 @@ class rapiddns(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result" - response = await self.request_with_fail_count(url) - return response + return await self.request_with_fail_count(url) def parse_results(self, r, query): results = set() diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index a8f57709a..1e047ccfc 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -27,9 +27,7 @@ async def setup(self): async def filter_event(self, event): if str(event.module) == "ipneighbor": return False - if getattr(event.host, "is_private", False): - return False - return True + return not getattr(event.host, "is_private", False) async def handle_event(self, event): host = event.host @@ -56,7 +54,7 @@ async def report(self): count = self.asn_counts[subnet] number = asn["asn"] if number != "UNKNOWN": - number = "AS" + number + number = f"AS{number}" name = asn["name"] country = asn["country"] description = asn["description"] @@ -89,7 +87,7 @@ async def get_asn(self, ip, retries=1): returns a list of ASNs, e.g.: [{'asn': '54113', 'subnet': '2606:50c0:8000::/48', 'name': 'FASTLY', 'description': 'Fastly', 'country': 'US', 'emails': []}, {'asn': '54113', 'subnet': '2606:50c0:8000::/46', 'name': 'FASTLY', 'description': 'Fastly', 'country': 'US', 'emails': []}] """ - for attempt in range(retries + 1): + for _ in range(retries + 1): for i, source in enumerate(list(self.sources)): get_asn_fn = getattr(self, f"get_asn_{source}") res = await get_asn_fn(ip) @@ -108,15 +106,11 @@ async def get_asn_ripe(self, ip): asns = [] if response == False: return False - data = response.get("data", {}) - if not data: - data = {} + data = response.get("data", {}) or {} prefix = data.get("prefix", "") asn_numbers = data.get("asns", []) if not prefix or not asn_numbers: return [] - if not asn_numbers: - asn_numbers = [] for number in asn_numbers: asn = await self.get_asn_metadata_ripe(number) if asn == False: @@ -135,14 +129,10 @@ async def get_asn_metadata_ripe(self, asn_number): response = await self.get_url(url, "ASN Metadata", cache=True) if response == False: return False - data = response.get("data", {}) - if not data: - data = {} - records = data.get("records", []) - if not records: - records = [] + data = response.get("data", {}) or {} + records = data.get("records", []) or [] emails = set() - asn = {k: "" for k in metadata_keys.keys()} + asn = {k: "" for k in metadata_keys} for record in records: for item in record: key = item.get("key", "") @@ -178,7 +168,7 @@ async def get_asn_bgpview(self, ip): description = details.get("description") or prefix.get("description") or "" country = details.get("country_code") or prefix.get("country_code") or "" emails = [] - if not asn in asns_tried: + if asn not in asns_tried: emails = await self.get_emails_bgpview(asn) if emails == False: return False @@ -213,9 +203,7 @@ async def get_url(self, url, data_type, cache=False): data = {} try: j = r.json() - if not isinstance(j, dict): - return data - return j + return j if isinstance(j, dict) else data except Exception as e: self.verbose(f"Error retrieving {data_type} at {url}: {e}", trace=True) self.debug(f"Got data: {getattr(r, 'content', '')}") diff --git a/bbot/modules/riddler.py b/bbot/modules/riddler.py index d525acbad..e96625b07 100644 --- a/bbot/modules/riddler.py +++ b/bbot/modules/riddler.py @@ -11,8 +11,7 @@ class riddler(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/search/exportcsv?q=pld:{self.helpers.quote(query)}" - response = await self.request_with_fail_count(url) - return response + return await self.request_with_fail_count(url) def parse_results(self, r, query): results = set() diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index 98b114b75..a279fd887 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -26,15 +26,13 @@ async def handle_event(self, event): url = f"{host}robots.txt" result = await self.helpers.request(url) if result: - body = result.text - - if body: + if body := result.text: lines = body.split("\n") for l in lines: if len(l) > 0: split_l = l.split(": ") if (split_l[0].lower() == "allow" and self.config.get("include_allow") == True) or ( - split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True + split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True ): unverified_url = f"{host}{split_l[1].lstrip('/')}".replace( "*", self.helpers.rand_string(4) diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py index b5115c1fa..2051789a7 100644 --- a/bbot/modules/secretsdb.py +++ b/bbot/modules/secretsdb.py @@ -46,11 +46,10 @@ async def handle_event(self, event): resp_headers = event.data.get("raw_header", "") all_matches = await self.scan.run_in_executor(self.search_data, resp_body, resp_headers) for matches, name in all_matches: - matches = [m.string[m.start() : m.end()] for m in matches] + matches = [m.string[m.start(): m.end()] for m in matches] description = f"Possible secret ({name}): {matches}" event_data = {"host": str(event.host), "description": description} - parsed_url = getattr(event, "parsed", None) - if parsed_url: + if parsed_url := getattr(event, "parsed", None): event_data["url"] = parsed_url.geturl() self.emit_event( event_data, @@ -65,7 +64,6 @@ def search_data(self, resp_body, resp_headers): name = r["name"] for text in (resp_body, resp_headers): if text: - matches = list(regex.finditer(text)) - if matches: + if matches := list(regex.finditer(text)): all_matches.append((matches, name)) return all_matches diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index d58b227e5..cc248127a 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -23,8 +23,7 @@ async def ping(self): async def request_url(self, query): url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}" - response = await self.request_with_fail_count(url) - return response + return await self.request_with_fail_count(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 7780120b6..7a13eb110 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -19,11 +19,9 @@ async def ping(self): async def request_url(self, query): url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}" - response = await self.request_with_fail_count(url) - return response + return await self.request_with_fail_count(url) def parse_results(self, r, query): - json = r.json() - if json: + if json := r.json(): for hostname in json.get("subdomains", []): yield f"{hostname}.{query}" diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py index 87358a955..84c192d17 100644 --- a/bbot/modules/sitedossier.py +++ b/bbot/modules/sitedossier.py @@ -18,7 +18,11 @@ async def handle_event(self, event): except ValueError as e: self.verbose(e) continue - if hostname and hostname.endswith(f".{query}") and not hostname == event.data: + if ( + hostname + and hostname.endswith(f".{query}") + and hostname != event.data + ): await self.emit_event_wait(hostname, "DNS_NAME", event, abort_if=self.abort_if) async def query(self, query, parse_fn=None, request_fn=None): @@ -26,7 +30,7 @@ async def query(self, query, parse_fn=None, request_fn=None): base_url = f"{self.base_url}/{self.helpers.quote(query)}" url = str(base_url) for i, page in enumerate(range(1, 100 * self.max_pages + 2, 100)): - self.verbose(f"Fetching page #{i+1} for {query}") + self.verbose(f"Fetching page #{i + 1} for {query}") if page > 1: url = f"{base_url}/{page}" response = await self.helpers.request(url) diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py index 71d0e883e..118e59c49 100644 --- a/bbot/modules/skymem.py +++ b/bbot/modules/skymem.py @@ -32,9 +32,9 @@ async def handle_event(self, event): continue for email in self.helpers.extract_emails(r2.text): self.emit_event(email, "EMAIL_ADDRESS", source=event) - pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text) + pages = re.findall(f"/domain/{domain_id}" + r"\?p=(\d+)", r2.text) if not pages: break - last_page = max([int(p) for p in pages]) + last_page = max(int(p) for p in pages) if page >= last_page: break diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py index d1996456b..f34378fc6 100644 --- a/bbot/modules/smuggler.py +++ b/bbot/modules/smuggler.py @@ -2,7 +2,6 @@ from bbot.modules.base import BaseModule - """ wrapper for https://github.com/defparam/smuggler.git """ diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 01dc9844b..f5b070190 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -44,11 +44,7 @@ async def filter_event(self, event): async def handle_event(self, event): _host = event.host - if event.port: - port = event.port - else: - port = 443 - + port = event.port or 443 # turn hostnames into IP address(es) if self.helpers.is_ip(_host): hosts = [_host] @@ -65,7 +61,7 @@ async def handle_event(self, event): tasks = [self.visit_host(host, port) for host in hosts] async for task in self.helpers.as_completed(tasks): result = await task - if not isinstance(result, tuple) or not len(result) == 3: + if not isinstance(result, tuple) or len(result) != 3: continue dns_names, emails, (host, port) = result if len(dns_names) > abort_threshold: @@ -79,8 +75,12 @@ async def handle_event(self, event): if event_data is not None and event_data != event: self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]") try: - ssl_event = self.make_event(event_data, event_type, source=event, raise_error=True) - if ssl_event: + if ssl_event := self.make_event( + event_data, + event_type, + source=event, + raise_error=True, + ): self.emit_event(ssl_event, on_success_callback=self.on_success_callback) except ValidationError as e: self.hugeinfo(f'Malformed {event_type} "{event_data}" at {event.data}') @@ -123,9 +123,7 @@ async def visit_host(self, host, port): self.debug(f"Timed out after {self.timeout} seconds while connecting to {netloc}") return [], [], (host, port) except Exception as e: - log_fn = self.warning - if isinstance(e, OSError): - log_fn = self.debug + log_fn = self.debug if isinstance(e, OSError) else self.warning log_fn(f"Error connecting to {netloc}: {e}") return [], [], (host, port) finally: @@ -168,7 +166,7 @@ def get_cert_sans(cert): sans = [] raw_sans = None ext_count = cert.get_extension_count() - for i in range(0, ext_count): + for i in range(ext_count): ext = cert.get_extension(i) short_name = str(ext.get_short_name()) if "subjectAltName" in short_name: diff --git a/bbot/modules/subdomain_hijack.py b/bbot/modules/subdomain_hijack.py index a25b1d64b..81c783e9b 100644 --- a/bbot/modules/subdomain_hijack.py +++ b/bbot/modules/subdomain_hijack.py @@ -45,19 +45,13 @@ async def handle_event(self, event): if hijackable: source_hosts = [] e = event - while 1: - host = getattr(e, "host", "") - if host: - if e not in source_hosts: - source_hosts.append(e) - e = e.get_source() - else: - break - + while 1 and (host := getattr(e, "host", "")): + if e not in source_hosts: + source_hosts.append(e) + e = e.get_source() url = f"https://{event.host}" description = f'Hijackable Subdomain "{event.data}": {reason}' - source_hosts = source_hosts[::-1] - if source_hosts: + if source_hosts := source_hosts[::-1]: source_hosts_str = str(source_hosts[0].host) for e in source_hosts[1:]: source_hosts_str += f" -[{e.module.name}]-> {e.host}" @@ -70,11 +64,11 @@ async def check_subdomain(self, event): for f in self.fingerprints: for domain in f.domains: self_matches = self.helpers.host_in_host(event.data, domain) - child_matches = any(self.helpers.host_in_host(domain, h) for h in event.resolved_hosts) if event.type == "DNS_NAME_UNRESOLVED": if self_matches and f.nxdomain: return True, "NXDOMAIN" else: + child_matches = any(self.helpers.host_in_host(domain, h) for h in event.resolved_hosts) if self_matches or child_matches: for scheme in ("https", "http"): if self.scan.stopping: @@ -104,14 +98,14 @@ async def _verify_fingerprint(self, fingerprint, *args, **kwargs): return True, f"HTTP status == {fingerprint.http_status}" text = getattr(r, "text", "") if ( - not fingerprint.nxdomain - and not fingerprint.http_status - and fingerprint.fingerprint_regex.findall(text) + not fingerprint.nxdomain + and not fingerprint.http_status + and fingerprint.fingerprint_regex.findall(text) ): return True, "Fingerprint match" except httpx.RequestError as e: if fingerprint.nxdomain and "Name or service not known" in str(e): - return True, f"NXDOMAIN" + return True, "NXDOMAIN" return False, "No match" @@ -120,7 +114,7 @@ def __init__(self, fingerprint): assert isinstance(fingerprint, dict), "fingerprint must be a dictionary" self.engine = fingerprint.get("service") self.cnames = fingerprint.get("cname", []) - self.domains = list(set([tldextract(c).registered_domain for c in self.cnames])) + self.domains = list({tldextract(c).registered_domain for c in self.cnames}) self.http_status = fingerprint.get("http_status", None) self.nxdomain = fingerprint.get("nxdomain", False) self.vulnerable = fingerprint.get("vulnerable", False) diff --git a/bbot/modules/subdomaincenter.py b/bbot/modules/subdomaincenter.py index 6d1825b8b..548f4e4f7 100644 --- a/bbot/modules/subdomaincenter.py +++ b/bbot/modules/subdomaincenter.py @@ -30,8 +30,5 @@ async def request_url(self, query): return response def parse_results(self, r, query): - results = set() json = r.json() - if json and isinstance(json, list): - results = set(json) - return results + return set(json) if json and isinstance(json, list) else set() diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index e012b6af6..9b87e317a 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -270,15 +270,9 @@ async def handle_event(self, event): async def test_detector(self, baseurl, detector): result = None - if "/" != baseurl[-1]: - url = f"{baseurl}/{detector}" - else: - url = f"{baseurl}{detector}" + url = f"{baseurl}/{detector}" if baseurl[-1] != "/" else f"{baseurl}{detector}" result = await self.helpers.request(url, timeout=self.timeout) return result, detector async def filter_event(self, event): - if "endpoint" in event.tags: - return False - else: - return True + return "endpoint" not in event.tags diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 61c2de6a4..b21498a5f 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -37,8 +37,12 @@ async def handle_event(self, event): except ValueError as e: self.verbose(e) continue - if hostname and hostname.endswith(f".{query}") and not hostname == event.data: - self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) + if ( + hostname + and hostname.endswith(f".{query}") + and hostname != event.data + ): + self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) async def request_url(self, query): url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}" @@ -52,10 +56,8 @@ def make_query(self, event): return ".".join([s for s in query.split(".") if s != "_wildcard"]) def parse_results(self, r, query=None): - json = r.json() - if json: - for hostname in json: - yield hostname + if json := r.json(): + yield from json async def query(self, query, parse_fn=None, request_fn=None): if parse_fn is None: @@ -109,10 +111,7 @@ async def eligible_for_enumeration(self, event): query = self.make_query(event) # check if wildcard is_wildcard = await self._is_wildcard(query) - # check if cloud - is_cloud = False - if any(t.startswith("cloud-") for t in event.tags): - is_cloud = True + is_cloud = any((t.startswith("cloud-") for t in event.tags)) # reject if it's a cloud resource and not in our target if is_cloud and event not in self.scan.target: return False, "Event is a cloud resource and not a direct target" @@ -120,27 +119,25 @@ async def eligible_for_enumeration(self, event): if self.reject_wildcards: if any(t in event.tags for t in ("a-error", "aaaa-error")): return False, "Event has a DNS resolution error" - if self.reject_wildcards == "strict": - if is_wildcard: - return False, "Event is a wildcard domain" - elif self.reject_wildcards == "cloud_only": + if self.reject_wildcards == "cloud_only": if is_wildcard and is_cloud: return False, "Event is both a cloud resource and a wildcard domain" + elif self.reject_wildcards == "strict": + if is_wildcard: + return False, "Event is a wildcard domain" return True, "" def already_processed(self, hostname): - for parent in self.helpers.domain_parents(hostname, include_self=True): - if hash(parent) in self.processed: - return True - return False + return any( + hash(parent) in self.processed + for parent in self.helpers.domain_parents(hostname, include_self=True) + ) async def abort_if(self, event): # this helps weed out unwanted results when scanning IP_RANGES and wildcard domains if "in-scope" not in event.tags: return True - if await self._is_wildcard(event.data): - return True - return False + return bool(await self._is_wildcard(event.data)) class subdomain_enum_apikey(subdomain_enum): diff --git a/bbot/modules/threatminer.py b/bbot/modules/threatminer.py index bbc1e23c3..58967ea13 100644 --- a/bbot/modules/threatminer.py +++ b/bbot/modules/threatminer.py @@ -13,8 +13,7 @@ class threatminer(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/domain.php?q={self.helpers.quote(query)}&rt=5" - r = await self.request_with_fail_count(url, timeout=self.http_timeout + 30) - return r + return await self.request_with_fail_count(url, timeout=self.http_timeout + 30) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index f4d598c63..2c3d4bb56 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -32,9 +32,14 @@ async def setup(self): ".pdf", ".gif", ] - for ext in extensions: - self.signatures.append(("GET", "{scheme}://{netloc}/{path}?%s=%s" % (self.rand_string, ext), False)) - + self.signatures.extend( + ( + "GET", + "{scheme}://{netloc}/{path}?%s=%s" % (self.rand_string, ext), + False, + ) + for ext in extensions + ) self.allow_redirects = self.config.get("allow_redirects", True) return True @@ -68,29 +73,26 @@ async def handle_event(self, event): if subject_response.text != None: subject_content += subject_response.text - if self.rand_string not in subject_content: - if match == False: - if str(subject_response.status_code).startswith("2"): - if "body" in reasons: - reported_signature = f"Modified URL: {sig[1]}" - description = f"Url Manipulation: [{','.join(reasons)}] Sig: [{reported_signature}]" - self.emit_event( - {"description": description, "host": str(event.host), "url": event.data}, - "FINDING", - source=event, - ) - else: - self.debug(f"Status code changed to {str(subject_response.status_code)}, ignoring") - else: + if self.rand_string in subject_content: self.debug("Ignoring positive result due to presence of parameter name in result") + elif match == False: + if str(subject_response.status_code).startswith("2"): + if "body" in reasons: + reported_signature = f"Modified URL: {sig[1]}" + description = f"Url Manipulation: [{','.join(reasons)}] Sig: [{reported_signature}]" + self.emit_event( + {"description": description, "host": str(event.host), "url": event.data}, + "FINDING", + source=event, + ) + else: + self.debug(f"Status code changed to {str(subject_response.status_code)}, ignoring") + async def filter_event(self, event): accepted_status_codes = ["200", "301", "302"] - for c in accepted_status_codes: - if f"status-{c}" in event.tags: - return True - return False + return any(f"status-{c}" in event.tags for c in accepted_status_codes) def format_signature(self, sig, event): if sig[2] == True: diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index f1efe08e5..0281bd800 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -22,14 +22,18 @@ async def handle_event(self, event): for domain, url in await self.query(query): source_event = event if domain and domain != query: - domain_event = self.make_event(domain, "DNS_NAME", source=event) - if domain_event: - if str(domain_event.host).endswith(query) and not str(domain_event.host) == str(event.host): + if domain_event := self.make_event( + domain, "DNS_NAME", source=event + ): + if str(domain_event.host).endswith(query) and str( + domain_event.host + ) != str(event.host): self.emit_event(domain_event, abort_if=self.abort_if) source_event = domain_event if url: - url_event = self.make_event(url, "URL_UNVERIFIED", source=source_event) - if url_event: + if url_event := self.make_event( + url, "URL_UNVERIFIED", source=source_event + ): if str(url_event.host).endswith(query): if self.urls: self.emit_event(url_event, abort_if=self.abort_if) @@ -62,5 +66,5 @@ async def query(self, query): else: self.debug(f'No results for "{query}"') except Exception: - self.verbose(f"Error retrieving urlscan results") + self.verbose("Error retrieving urlscan results") return results diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py index c2a5e4431..44e6a9249 100644 --- a/bbot/modules/viewdns.py +++ b/bbot/modules/viewdns.py @@ -50,7 +50,7 @@ async def query(self, query): domain = table_cells[0].text.strip().lower() # registrar == last cell registrar = table_cells[-1].text.strip() - if domain and not domain == query: + if domain and domain != query: result = (domain, registrar) result_hash = hash(result) if result_hash not in found: diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 90575aaa7..3ebef9b0e 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -37,10 +37,10 @@ async def query(self, query): ) try: async for response in agen: - r = self.parse_results(response, query) - if not r: + if r := self.parse_results(response, query): + results.update(r) + else: break - results.update(r) finally: agen.aclose() return results diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index f15b82263..cf5e71661 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -27,17 +27,16 @@ async def handle_event(self, event): if waf_detections: for waf in waf_detections: self.emit_event({"host": str(event.host), "url": url, "WAF": waf}, "WAF", source=event) - else: - if self.config.get("generic_detect") == True: - generic = await self.scan.run_in_executor(WW.genericdetect) - if generic: - self.emit_event( - { - "host": str(event.host), - "url": url, - "WAF": "generic detection", - "info": WW.knowledge["generic"]["reason"], - }, - "WAF", - source=event, - ) + elif self.config.get("generic_detect") == True: + generic = await self.scan.run_in_executor(WW.genericdetect) + if generic: + self.emit_event( + { + "host": str(event.host), + "url": url, + "WAF": "generic detection", + "info": WW.knowledge["generic"]["reason"], + }, + "WAF", + source=event, + ) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 572ce4cd1..01ceed63a 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -61,7 +61,7 @@ async def init_events(self): - It also marks the Scan object as finished with initialization by setting `_finished_init` to True. """ - context = f"manager.init_events()" + context = "manager.init_events()" async with self.scan._acatch(context), self._task_counter.count(context): await self.distribute_event(self.scan.root_event) sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) @@ -86,9 +86,7 @@ async def emit_event(self, event, *args, **kwargs): log.debug(f'Module "{event.module}" raised {event}') - # "quick" queues the event immediately - quick = kwargs.pop("quick", False) - if quick: + if quick := kwargs.pop("quick", False): log.debug(f'Module "{event.module}" raised {event}') event._resolved.set() for kwarg in ["abort_if", "on_success_callback"]: @@ -199,8 +197,9 @@ async def _emit_event(self, event, **kwargs): event._resolved_hosts = resolved_hosts event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event) - event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event) - if event_blacklisted: + if event_blacklisted := event_blacklisted_dns | self.scan.blacklisted( + event + ): event.add_tag("blacklisted") # Blacklist purging @@ -212,7 +211,11 @@ async def _emit_event(self, event, **kwargs): distribute_event = False # DNS_NAME --> DNS_NAME_UNRESOLVED - if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: + if ( + event.type == "DNS_NAME" + and "unresolved" in event.tags + and "target" not in event.tags + ): event.type = "DNS_NAME_UNRESOLVED" # Cloud tagging @@ -222,11 +225,9 @@ async def _emit_event(self, event, **kwargs): # Scope shepherding # here, we buff or nerf the scope distance of an event based on its attributes and certain scan settings event_is_duplicate = self.is_duplicate_event(event) - event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance - set_scope_distance = event.scope_distance - if event_whitelisted: - set_scope_distance = 0 + set_scope_distance = 0 if event_whitelisted else event.scope_distance if event.host: + event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance # here, we evaluate some weird logic # the reason this exists is to ensure we don't have orphans in the graph # because forcefully internalizing certain events can orphan their children @@ -239,18 +240,19 @@ async def _emit_event(self, event, **kwargs): # force re-emit internal source events for s in source_trail: self.queue_event(s) - else: - if event.scope_distance > self.scan.scope_report_distance: - log.debug( - f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" - ) - event.make_internal() + elif event.scope_distance > self.scan.scope_report_distance: + log.debug( + f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" + ) + event.make_internal() # check for wildcards - if event.scope_distance <= self.scan.scope_search_distance: - if not "unresolved" in event.tags: - if not self.scan.helpers.is_ip_type(event.host): - await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) + if ( + event.scope_distance <= self.scan.scope_search_distance + and "unresolved" not in event.tags + and not self.scan.helpers.is_ip_type(event.host) + ): + await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) # now that the event is properly tagged, we can finally make decisions about it abort_result = False @@ -269,10 +271,9 @@ async def _emit_event(self, event, **kwargs): return # run success callback before distributing event (so it can add tags, etc.) - if distribute_event: - if callable(on_success_callback): - async with self.scan._acatch(context=on_success_callback): - await self.scan.helpers.execute_sync_or_async(on_success_callback, event) + if distribute_event and callable(on_success_callback): + async with self.scan._acatch(context=on_success_callback): + await self.scan.helpers.execute_sync_or_async(on_success_callback, event) if not event.host or (event.always_emit and not event_is_duplicate): log.debug( @@ -289,9 +290,15 @@ async def _emit_event(self, event, **kwargs): # speculate DNS_NAMES and IP_ADDRESSes from other event types source_event = event if ( - event.host - and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") - and not str(event.module) == "speculate" + event.host + and event.type + not in ( + "DNS_NAME", + "DNS_NAME_UNRESOLVED", + "IP_ADDRESS", + "IP_RANGE", + ) + and str(event.module) != "speculate" ): source_module = self.scan.helpers._make_dummy_module("host", _type="internal") source_module._priority = 4 @@ -533,7 +540,7 @@ def modules_status(self, _log=False): ) if event_type_summary: self.scan.info( - f'{self.scan.name}: Events produced so far: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}' + f'{self.scan.name}: Events produced so far: {", ".join([f"{k}: {v}" for k, v in event_type_summary])}' ) else: self.scan.info(f"{self.scan.name}: No events produced yet") diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index ecc4d31e4..53649365d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -113,19 +113,19 @@ class Scanner: } def __init__( - self, - *targets, - whitelist=None, - blacklist=None, - scan_id=None, - name=None, - modules=None, - output_modules=None, - output_dir=None, - config=None, - dispatcher=None, - strict_scope=False, - force_start=False, + self, + *targets, + whitelist=None, + blacklist=None, + scan_id=None, + name=None, + modules=None, + output_modules=None, + output_dir=None, + config=None, + dispatcher=None, + strict_scope=False, + force_start=False, ): """ Initializes the Scanner class. @@ -154,10 +154,7 @@ def __init__( if isinstance(output_modules, str): output_modules = [output_modules] - if config is None: - config = OmegaConf.create({}) - else: - config = OmegaConf.create(config) + config = OmegaConf.create({}) if config is None else OmegaConf.create(config) self.config = OmegaConf.merge(bbot_config, config) prepare_environment(self.config) if self.config.get("debug", False): @@ -214,10 +211,7 @@ def __init__( blacklist = [] self.blacklist = Target(self, *blacklist) - if dispatcher is None: - self.dispatcher = Dispatcher() - else: - self.dispatcher = dispatcher + self.dispatcher = Dispatcher() if dispatcher is None else dispatcher self.dispatcher.set_scan(self) self.manager = ScanManager(self) @@ -254,7 +248,9 @@ def __init__( try: mp.set_start_method("spawn") except Exception: - self.warning(f"Failed to set multiprocessing spawn method. This may negatively affect performance.") + self.warning( + "Failed to set multiprocessing spawn method. This may negatively affect performance." + ) self.process_pool = ProcessPoolExecutor() self._stopping = False @@ -285,18 +281,17 @@ async def _prep(self): await self.load_modules() - self.info(f"Setting up modules...") + self.info("Setting up modules...") await self.setup_modules() self.success(f"Setup succeeded for {len(self.modules):,} modules.") self._prepped = True def start(self): - for event in async_to_sync_gen(self.async_start()): - yield event + yield from async_to_sync_gen(self.async_start()) def start_without_generator(self): - for event in async_to_sync_gen(self.async_start()): + for _ in async_to_sync_gen(self.async_start()): pass async def async_start_without_generator(self): @@ -313,7 +308,7 @@ async def async_start(self): self._start_log_handlers() if not self.target: - self.warning(f"No scan targets specified") + self.warning("No scan targets specified") # start status ticker self.ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) @@ -321,7 +316,7 @@ async def async_start(self): self.status = "STARTING" if not self.modules: - self.error(f"No modules loaded") + self.error("No modules loaded") self.status = "FAILED" return else: @@ -404,7 +399,7 @@ async def async_start(self): self._stop_log_handlers() def _start_modules(self): - self.verbose(f"Starting module worker loops") + self.verbose("Starting module worker loops") for module_name, module in self.modules.items(): module.start() @@ -428,7 +423,7 @@ async def setup_modules(self, remove_failed=True): Soft-failed modules are not set to an error state but are also removed if `remove_failed` is True. """ await self.load_modules() - self.verbose(f"Setting up modules") + self.verbose("Setting up modules") succeeded = [] hard_failed = [] soft_failed = [] @@ -491,64 +486,65 @@ async def load_modules(self): Note: After all modules are loaded, they are sorted by `_priority` and stored in the `modules` dictionary. """ - if not self._modules_loaded: - all_modules = list(set(self._scan_modules + self._output_modules + self._internal_modules)) - if not all_modules: - self.warning(f"No modules to load") - return - - if not self._scan_modules: - self.warning(f"No scan modules to load") - - # install module dependencies - succeeded, failed = await self.helpers.depsinstaller.install( - *self._scan_modules, *self._output_modules, *self._internal_modules + if self._modules_loaded: + return + all_modules = list(set(self._scan_modules + self._output_modules + self._internal_modules)) + if not all_modules: + self.warning("No modules to load") + return + + if not self._scan_modules: + self.warning("No scan modules to load") + + # install module dependencies + succeeded, failed = await self.helpers.depsinstaller.install( + *self._scan_modules, *self._output_modules, *self._internal_modules + ) + if failed: + msg = f"Failed to install dependencies for {len(failed):,} modules: {','.join(failed)}" + self._fail_setup(msg) + modules = sorted([m for m in self._scan_modules if m in succeeded]) + output_modules = sorted([m for m in self._output_modules if m in succeeded]) + internal_modules = sorted([m for m in self._internal_modules if m in succeeded]) + + # Load scan modules + self.verbose(f"Loading {len(modules):,} scan modules: {','.join(modules)}") + loaded_modules, failed = self._load_modules(modules) + self.modules.update(loaded_modules) + if len(failed) > 0: + msg = f"Failed to load {len(failed):,} scan modules: {','.join(failed)}" + self._fail_setup(msg) + if loaded_modules: + self.info( + f"Loaded {len(loaded_modules):,}/{len(self._scan_modules):,} scan modules ({','.join(loaded_modules)})" ) - if failed: - msg = f"Failed to install dependencies for {len(failed):,} modules: {','.join(failed)}" - self._fail_setup(msg) - modules = sorted([m for m in self._scan_modules if m in succeeded]) - output_modules = sorted([m for m in self._output_modules if m in succeeded]) - internal_modules = sorted([m for m in self._internal_modules if m in succeeded]) - - # Load scan modules - self.verbose(f"Loading {len(modules):,} scan modules: {','.join(modules)}") - loaded_modules, failed = self._load_modules(modules) - self.modules.update(loaded_modules) - if len(failed) > 0: - msg = f"Failed to load {len(failed):,} scan modules: {','.join(failed)}" - self._fail_setup(msg) - if loaded_modules: - self.info( - f"Loaded {len(loaded_modules):,}/{len(self._scan_modules):,} scan modules ({','.join(loaded_modules)})" - ) - # Load internal modules - self.verbose(f"Loading {len(internal_modules):,} internal modules: {','.join(internal_modules)}") - loaded_internal_modules, failed_internal = self._load_modules(internal_modules) - self.modules.update(loaded_internal_modules) - if len(failed_internal) > 0: - msg = f"Failed to load {len(loaded_internal_modules):,} internal modules: {','.join(loaded_internal_modules)}" - self._fail_setup(msg) - if loaded_internal_modules: - self.info( - f"Loaded {len(loaded_internal_modules):,}/{len(self._internal_modules):,} internal modules ({','.join(loaded_internal_modules)})" - ) + # Load internal modules + self.verbose(f"Loading {len(internal_modules):,} internal modules: {','.join(internal_modules)}") + loaded_internal_modules, failed_internal = self._load_modules(internal_modules) + self.modules.update(loaded_internal_modules) + if len(failed_internal) > 0: + msg = f"Failed to load {len(loaded_internal_modules):,} internal modules: {','.join(loaded_internal_modules)}" + self._fail_setup(msg) + if loaded_internal_modules: + self.info( + f"Loaded {len(loaded_internal_modules):,}/{len(self._internal_modules):,} internal modules ({','.join(loaded_internal_modules)})" + ) - # Load output modules - self.verbose(f"Loading {len(output_modules):,} output modules: {','.join(output_modules)}") - loaded_output_modules, failed_output = self._load_modules(output_modules) - self.modules.update(loaded_output_modules) - if len(failed_output) > 0: - msg = f"Failed to load {len(failed_output):,} output modules: {','.join(failed_output)}" - self._fail_setup(msg) - if loaded_output_modules: - self.info( - f"Loaded {len(loaded_output_modules):,}/{len(self._output_modules):,} output modules, ({','.join(loaded_output_modules)})" - ) + # Load output modules + self.verbose(f"Loading {len(output_modules):,} output modules: {','.join(output_modules)}") + loaded_output_modules, failed_output = self._load_modules(output_modules) + self.modules.update(loaded_output_modules) + if len(failed_output) > 0: + msg = f"Failed to load {len(failed_output):,} output modules: {','.join(failed_output)}" + self._fail_setup(msg) + if loaded_output_modules: + self.info( + f"Loaded {len(loaded_output_modules):,}/{len(self._output_modules):,} output modules, ({','.join(loaded_output_modules)})" + ) - self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "_priority", 0))) - self._modules_loaded = True + self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], "_priority", 0))) + self._modules_loaded = True def stop(self): """Stops the in-progress scan and performs necessary cleanup. @@ -561,7 +557,7 @@ def stop(self): if not self._stopping: self._stopping = True self.status = "ABORTING" - self.hugewarning(f"Aborting scan") + self.hugewarning("Aborting scan") self.trace() self._cancel_tasks() self._drain_queues() @@ -752,29 +748,27 @@ def status(self, status): """ status = str(status).strip().upper() if status in self._status_codes: - if self.status == "ABORTING" and not status == "ABORTED": + if self.status == "ABORTING" and status != "ABORTED": self.debug(f'Attempt to set invalid status "{status}" on aborted scan') + elif status == self._status: + self.debug(f'Scan status is already "{status}"') else: - if status != self._status: - self._status = status - self._status_code = self._status_codes[status] - self.dispatcher_tasks.append( - asyncio.create_task(self.dispatcher.catch(self.dispatcher.on_status, self._status, self.id)) - ) - else: - self.debug(f'Scan status is already "{status}"') + self._status = status + self._status_code = self._status_codes[status] + self.dispatcher_tasks.append( + asyncio.create_task(self.dispatcher.catch(self.dispatcher.on_status, self._status, self.id)) + ) else: self.debug(f'Attempt to set invalid status "{status}" on scan') def make_event(self, *args, **kwargs): kwargs["scan"] = self - event = make_event(*args, **kwargs) - return event + return make_event(*args, **kwargs) @property def log(self): if self._log is None: - self._log = logging.getLogger(f"bbot.agent.scanner") + self._log = logging.getLogger("bbot.agent.scanner") return self._log @property @@ -842,14 +836,23 @@ def dns_regexes(self): ... hostname = match.group().lower() """ if self._dns_regexes is None: - dns_targets = set(t.host for t in self.target if t.host and isinstance(t.host, str)) - dns_whitelist = set(t.host for t in self.whitelist if t.host and isinstance(t.host, str)) + dns_targets = { + t.host for t in self.target if t.host and isinstance(t.host, str) + } + dns_whitelist = { + t.host + for t in self.whitelist + if t.host and isinstance(t.host, str) + } dns_targets.update(dns_whitelist) dns_targets = sorted(dns_targets, key=len) dns_targets_set = set() dns_regexes = [] for t in dns_targets: - if not any(x in dns_targets_set for x in self.helpers.domain_parents(t, include_self=True)): + if all( + x not in dns_targets_set + for x in self.helpers.domain_parents(t, include_self=True) + ): dns_targets_set.add(t) dns_regexes.append(re.compile(r"((?:(?:[\w-]+)\.)+" + re.escape(t) + ")", re.I)) self._dns_regexes = dns_regexes @@ -868,19 +871,18 @@ def json(self): """ A dictionary representation of the scan including its name, ID, targets, whitelist, blacklist, and modules """ - j = dict() + j = {} for i in ("id", "name"): - v = getattr(self, i, "") - if v: - j.update({i: v}) + if v := getattr(self, i, ""): + j[i] = v if self.target: - j.update({"targets": [str(e.data) for e in self.target]}) + j["targets"] = [str(e.data) for e in self.target] if self.whitelist: - j.update({"whitelist": [str(e.data) for e in self.whitelist]}) + j["whitelist"] = [str(e.data) for e in self.whitelist] if self.blacklist: - j.update({"blacklist": [str(e.data) for e in self.blacklist]}) + j["blacklist"] = [str(e.data) for e in self.blacklist] if self.modules: - j.update({"modules": [str(m) for m in self.modules]}) + j["modules"] = [str(m) for m in self.modules] return j def debug(self, *args, trace=False, **kwargs): @@ -1061,7 +1063,7 @@ def _handle_exception(self, e, context="scan", finally_callback=None): filename, lineno, funcname = self.helpers.get_traceback_details(e) exception_chain = self.helpers.get_exception_chain(e) if any(isinstance(exc, KeyboardInterrupt) for exc in exception_chain): - log.debug(f"Interrupted") + log.debug("Interrupted") self.stop() elif isinstance(e, BrokenPipeError): log.debug(f"BrokenPipeError in {filename}:{lineno}:{funcname}(): {e}") diff --git a/bbot/scanner/stats.py b/bbot/scanner/stats.py index cc2b3e576..9b7ace89e 100644 --- a/bbot/scanner/stats.py +++ b/bbot/scanner/stats.py @@ -49,16 +49,12 @@ def table(self): for mname, mstat in self.module_stats.items(): if mname == "TARGET" or mstat.module._stats_exclude: continue - table_row = [] - table_row.append(mname) produced_str = f"{mstat.produced_total:,}" - produced = sorted(mstat.produced.items(), key=lambda x: x[0]) - if produced: + if produced := sorted(mstat.produced.items(), key=lambda x: x[0]): produced_str += " (" + ", ".join(f"{c:,} {t}" for t, c in produced) + ")" - table_row.append(produced_str) + table_row = [mname, produced_str] consumed_str = f"{mstat.consumed_total:,}" - consumed = sorted(mstat.consumed.items(), key=lambda x: x[0]) - if consumed: + if consumed := sorted(mstat.consumed.items(), key=lambda x: x[0]): consumed_str += " (" + ", ".join(f"{c:,} {t}" for t, c in consumed) + ")" table_row.append(consumed_str) table.append(table_row) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index f733f8295..a953242f5 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -88,8 +88,8 @@ def __init__(self, scan, *targets, strict_scope=False, make_in_scope=False): self.make_in_scope = make_in_scope self._dummy_module = TargetDummyModule(scan) - self._events = dict() - if len(targets) > 0: + self._events = {} + if targets: log.verbose(f"Creating events from {len(targets):,} targets") for t in targets: self.add_target(t) @@ -222,9 +222,7 @@ def get(self, host): return next(iter(self._events[h])) def _contains(self, other): - if self.get(other) is not None: - return True - return False + return self.get(other) is not None def __str__(self): return ",".join([str(e.data) for e in self.events][:5]) @@ -233,12 +231,10 @@ def __iter__(self): yield from self.events def __contains__(self, other): - # if "other" is a Target - if type(other) == self.__class__: - contained_in_self = [self._contains(e) for e in other.events] - return all(contained_in_self) - else: + if type(other) != self.__class__: return self._contains(other) + contained_in_self = [self._contains(e) for e in other.events] + return all(contained_in_self) def __bool__(self): return bool(self._events) @@ -268,13 +264,12 @@ def __len__(self): - If a host is represented as an IP network, all individual IP addresses in that network are counted. - For other types of hosts, each unique event is counted as one. """ - num_hosts = 0 - for host, _events in self._events.items(): - if type(host) in (ipaddress.IPv4Network, ipaddress.IPv6Network): - num_hosts += host.num_addresses - else: - num_hosts += len(_events) - return num_hosts + return sum( + host.num_addresses + if type(host) in (ipaddress.IPv4Network, ipaddress.IPv6Network) + else len(_events) + for host, _events in self._events.items() + ) class TargetDummyModule(BaseModule): diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index b66488159..f9d179275 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -10,7 +10,6 @@ os.environ["BBOT_TABLE_FORMAT"] = "github" - # Make a regex pattern which will match any group of non-space characters that include a blacklisted character blacklist_chars = ["<", ">"] blacklist_re = re.compile(r"\|([^|]*[" + re.escape("".join(blacklist_chars)) + r"][^|]*)\|") @@ -19,9 +18,7 @@ def enclose_tags(text): - # Use re.sub() to replace matched words with the same words enclosed in backticks - result = blacklist_re.sub(r"|`\1`|", text) - return result + return blacklist_re.sub(r"|`\1`|", text) def find_replace_markdown(content, keyword, replace): @@ -43,10 +40,9 @@ def find_replace_file(file, keyword, replace): with open(file) as f: content = f.read() new_content = find_replace_markdown(content, keyword, replace) - if new_content != content: - if not "BBOT_TESTING" in os.environ: - with open(file, "w") as f: - f.write(new_content) + if new_content != content and "BBOT_TESTING" not in os.environ: + with open(file, "w") as f: + f.write(new_content) def update_docs(): @@ -96,8 +92,7 @@ def update_md_files(keyword, s): # Default config default_config_file = bbot_code_dir / "bbot" / "defaults.yml" - with open(default_config_file) as f: - default_config_yml = f.read() + default_config_yml = Path(default_config_file).read_text() default_config_yml = f'```yaml title="defaults.yml"\n{default_config_yml}\n```' assert len(default_config_yml.splitlines()) > 20 update_md_files("BBOT DEFAULT CONFIG", default_config_yml) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 2c74fe190..b4a659ac1 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -14,14 +14,11 @@ class SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher): def match_data(self, request: Request) -> bool: - if self.data is None: - return True - return self.data in request.data + return True if self.data is None else self.data in request.data pytest_httpserver.httpserver.RequestMatcher = SubstringRequestMatcher - test_config = OmegaConf.load(Path(__file__).parent / "test.conf") if test_config.get("debug", False): os.environ["BBOT_DEBUG"] = "True" @@ -41,7 +38,7 @@ def match_data(self, request: Request) -> bool: tldextract.extract("www.evilcorp.com") -log = logging.getLogger(f"bbot.test.fixtures") +log = logging.getLogger("bbot.test.fixtures") @pytest.fixture diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 67e7515ff..95a396cb8 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -76,8 +76,7 @@ def bbot_httpserver_ssl(): @pytest.fixture def interactsh_mock_instance(): - interactsh_mock = Interactsh_mock() - return interactsh_mock + return Interactsh_mock() class Interactsh_mock: @@ -95,10 +94,13 @@ async def deregister(self, callback=None): pass async def poll(self): - poll_results = [] - for subdomain_tag in self.interactions: - poll_results.append({"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"}) - return poll_results + return [ + { + "full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", + "protocol": "HTTP", + } + for subdomain_tag in self.interactions + ] import threading diff --git a/bbot/test/test_step_1/test_agent.py b/bbot/test/test_step_1/test_agent.py index 73bb50355..c7deab236 100644 --- a/bbot/test/test_step_1/test_agent.py +++ b/bbot/test/test_step_1/test_agent.py @@ -4,7 +4,6 @@ from ..bbot_fixtures import * # noqa: F401 - _first_run = True success = False diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 0ccc94887..03d573833 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -1,3 +1,4 @@ +import contextlib from ..bbot_fixtures import * @@ -53,11 +54,8 @@ async def test_cli(monkeypatch, bbot_config): task = asyncio.create_task(cli._main()) await asyncio.sleep(2) task.cancel() - try: + with contextlib.suppress(asyncio.CancelledError): await task - except asyncio.CancelledError: - pass - # no args monkeypatch.setattr("sys.argv", ["bbot"]) await cli._main() diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py index 8827bcdad..89fbd94e1 100644 --- a/bbot/test/test_step_1/test_command.py +++ b/bbot/test/test_step_1/test_command.py @@ -75,13 +75,13 @@ async def test_command(bbot_scanner, bbot_config): # test piping lines = [] async for line in scan1.helpers.run_live( - ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"]) + ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"]) ): lines.append(line) assert lines == ["some", "random", "stdin"] lines = [] async for line in scan1.helpers.run_live( - ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"], text=False), text=False + ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"], text=False), text=False ): lines.append(line) assert lines == [b"some", b"random", b"stdin"] diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 09c3a4d2b..cc21828c8 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -111,9 +111,9 @@ async def test_wildcards(bbot_scanner, bbot_config): assert "SRV" not in wildcard_rdtypes assert wildcard_rdtypes["A"] == (True, "github.io") assert hash("github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.asdf.github.io") in helpers.dns._wildcard_cache - assert not hash("asdf.asdf.asdf.github.io") in helpers.dns._wildcard_cache + assert hash("asdf.github.io") not in helpers.dns._wildcard_cache + assert hash("asdf.asdf.github.io") not in helpers.dns._wildcard_cache + assert hash("asdf.asdf.asdf.github.io") not in helpers.dns._wildcard_cache assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", dummy=True) wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", dummy=True) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 842b91f9c..218916697 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -286,8 +286,8 @@ async def test_events(events, scan, helpers, bbot_config): assert scan.make_event("xn--eckwd4c7c.xn--zckzah:80", dummy=True).data == "xn--eckwd4c7c.xn--zckzah:80" assert scan.make_event("http://xn--eckwd4c7c.xn--zckzah:80", dummy=True).data == "http://xn--eckwd4c7c.xn--zckzah/" assert ( - scan.make_event("http://xn--eckwd4c7c.xn--zckzah:80/テスト", dummy=True).data - == "http://xn--eckwd4c7c.xn--zckzah/テスト" + scan.make_event("http://xn--eckwd4c7c.xn--zckzah:80/テスト", dummy=True).data + == "http://xn--eckwd4c7c.xn--zckzah/テスト" ) assert scan.make_event("ドメイン.テスト", dummy=True).data == "xn--eckwd4c7c.xn--zckzah" @@ -298,27 +298,28 @@ async def test_events(events, scan, helpers, bbot_config): assert scan.make_event("http://ドメイン.テスト:80/テスト", dummy=True).data == "http://xn--eckwd4c7c.xn--zckzah/テスト" # thai assert ( - scan.make_event("xn--12c1bik6bbd8ab6hd1b5jc6jta.com", dummy=True).data == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com" + scan.make_event("xn--12c1bik6bbd8ab6hd1b5jc6jta.com", + dummy=True).data == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com" ) assert ( - scan.make_event("bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com", dummy=True).data - == "bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com" + scan.make_event("bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com", dummy=True).data + == "bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com" ) assert ( - scan.make_event("ทดสอบ@xn--12c1bik6bbd8ab6hd1b5jc6jta.com", dummy=True).data - == "ทดสอบ@xn--12c1bik6bbd8ab6hd1b5jc6jta.com" + scan.make_event("ทดสอบ@xn--12c1bik6bbd8ab6hd1b5jc6jta.com", dummy=True).data + == "ทดสอบ@xn--12c1bik6bbd8ab6hd1b5jc6jta.com" ) assert ( - scan.make_event("xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80", dummy=True).data - == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80" + scan.make_event("xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80", dummy=True).data + == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80" ) assert ( - scan.make_event("http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80", dummy=True).data - == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/" + scan.make_event("http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80", dummy=True).data + == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/" ) assert ( - scan.make_event("http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80/ทดสอบ", dummy=True).data - == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ" + scan.make_event("http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80/ทดสอบ", dummy=True).data + == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ" ) assert scan.make_event("เราเที่ยวด้วยกัน.com", dummy=True).data == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com" @@ -326,12 +327,12 @@ async def test_events(events, scan, helpers, bbot_config): assert scan.make_event("ทดสอบ@เราเที่ยวด้วยกัน.com", dummy=True).data == "ทดสอบ@xn--12c1bik6bbd8ab6hd1b5jc6jta.com" assert scan.make_event("เราเที่ยวด้วยกัน.com:80", dummy=True).data == "xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80" assert ( - scan.make_event("http://เราเที่ยวด้วยกัน.com:80", dummy=True).data - == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/" + scan.make_event("http://เราเที่ยวด้วยกัน.com:80", dummy=True).data + == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/" ) assert ( - scan.make_event("http://เราเที่ยวด้วยกัน.com:80/ทดสอบ", dummy=True).data - == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ" + scan.make_event("http://เราเที่ยวด้วยกัน.com:80/ทดสอบ", dummy=True).data + == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ" ) # test event serialization diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index b27ab4577..d7d4bf42c 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -261,7 +261,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https fuzzy=True, exclude_keys="modules", ) - assert not "secrets_db" in filtered_dict4["modules"] + assert "secrets_db" not in filtered_dict4["modules"] assert "ipneighbor" in filtered_dict4["modules"] assert "secret" in filtered_dict4["modules"]["ipneighbor"] assert "asdf" not in filtered_dict4["modules"]["ipneighbor"] @@ -372,8 +372,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https # urls assert helpers.validators.validate_url(" httP://evilcorP.com/asdf?a=b&c=d#e") == "http://evilcorp.com/asdf" assert ( - helpers.validators.validate_url_parsed(" httP://evilcorP.com/asdf?a=b&c=d#e").geturl() - == "http://evilcorp.com/asdf" + helpers.validators.validate_url_parsed(" httP://evilcorP.com/asdf?a=b&c=d#e").geturl() + == "http://evilcorp.com/asdf" ) assert helpers.validators.soft_validate(" httP://evilcorP.com/asdf?a=b&c=d#e", "url") == True assert helpers.validators.soft_validate("!@#$", "url") == False @@ -409,15 +409,16 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https assert helpers.recursive_decode("%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021") == " Привет!" assert helpers.recursive_decode("Hello%2520world%2521") == "Hello world!" assert ( - helpers.recursive_decode("Hello%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442") - == "Hello Привет" + helpers.recursive_decode("Hello%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442") + == "Hello Привет" ) assert ( - helpers.recursive_decode("%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442%255Cu0021") - == " Привет!" + helpers.recursive_decode("%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442%255Cu0021") + == " Привет!" ) assert ( - helpers.recursive_decode(r"Hello\\nWorld\\\tGreetings\\\\nMore\nText") == "Hello\nWorld\tGreetings\nMore\nText" + helpers.recursive_decode( + r"Hello\\nWorld\\\tGreetings\\\\nMore\nText") == "Hello\nWorld\tGreetings\nMore\nText" ) ### CACHE ### @@ -497,30 +498,29 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https items = ["a", "b", "c", "d", "e"] first_frequencies = {i: 0 for i in items} weights = [1, 2, 3, 4, 5] - for i in range(10000): + for _ in range(10000): shuffled = helpers.weighted_shuffle(items, weights) first = shuffled[0] first_frequencies[first] += 1 assert ( - first_frequencies["a"] - < first_frequencies["b"] - < first_frequencies["c"] - < first_frequencies["d"] - < first_frequencies["e"] + first_frequencies["a"] + < first_frequencies["b"] + < first_frequencies["c"] + < first_frequencies["d"] + < first_frequencies["e"] ) def test_word_cloud(helpers, bbot_config, bbot_scanner): number_mutations = helpers.word_cloud.get_number_mutations("base2_p013", n=5, padding=2) - assert "base0_p013" in number_mutations - assert "base7_p013" in number_mutations - assert "base8_p013" not in number_mutations + assert_number_mutations_presence( + "base0_p013", number_mutations, "base7_p013", "base8_p013" + ) assert "base2_p008" in number_mutations assert "base2_p007" not in number_mutations - assert "base2_p018" in number_mutations - assert "base2_p0134" in number_mutations - assert "base2_p0135" not in number_mutations - + assert_number_mutations_presence( + "base2_p018", number_mutations, "base2_p0134", "base2_p0135" + ) permutations = helpers.word_cloud.mutations("_base", numbers=1) assert ("_base", "dev") in permutations assert ("dev", "_base") in permutations @@ -528,20 +528,12 @@ def test_word_cloud(helpers, bbot_config, bbot_scanner): # saving and loading scan1 = bbot_scanner("127.0.0.1", config=bbot_config) word_cloud = scan1.helpers.word_cloud - word_cloud.add_word("lantern") - word_cloud.add_word("black") - word_cloud.add_word("black") + add_words_to_mutator(word_cloud, "lantern", "black", "black") word_cloud.save() - with open(word_cloud.default_filename) as f: - word_cloud_content = [l.rstrip() for l in f.read().splitlines()] - assert len(word_cloud_content) == 2 - assert "2\tblack" in word_cloud_content + word_cloud_content = assert_word_cloud_file_content(word_cloud, 2) assert "1\tlantern" in word_cloud_content word_cloud.save(limit=1) - with open(word_cloud.default_filename) as f: - word_cloud_content = [l.rstrip() for l in f.read().splitlines()] - assert len(word_cloud_content) == 1 - assert "2\tblack" in word_cloud_content + word_cloud_content = assert_word_cloud_file_content(word_cloud, 1) assert "1\tlantern" not in word_cloud_content word_cloud.clear() with open(word_cloud.default_filename, "w") as f: @@ -570,9 +562,7 @@ def test_word_cloud(helpers, bbot_config, bbot_scanner): } m = DNSMutator() - m.add_word("blacklantern-security") - m.add_word("sec") - m.add_word("sec2") + add_words_to_mutator(m, "blacklantern-security", "sec", "sec2") m.add_word("black2") mutations = sorted(m.mutations("whitebasket")) assert mutations == sorted( @@ -610,6 +600,26 @@ def test_word_cloud(helpers, bbot_config, bbot_scanner): assert top_mutations[:2] == [((None,), 4), ((None, "2"), 2)] +def assert_number_mutations_presence(arg0, number_mutations, arg2, arg3): + assert arg0 in number_mutations + assert arg2 in number_mutations + assert arg3 not in number_mutations + + +def add_words_to_mutator(arg0, arg1, arg2, arg3): + arg0.add_word(arg1) + arg0.add_word(arg2) + arg0.add_word(arg3) + + +def assert_word_cloud_file_content(word_cloud, arg1): + with open(word_cloud.default_filename) as f: + result = [l.rstrip() for l in f.read().splitlines()] + assert len(result) == arg1 + assert "2\tblack" in result + return result + + def test_names(helpers): assert helpers.names == sorted(helpers.names) assert helpers.adjectives == sorted(helpers.adjectives) diff --git a/bbot/test/test_step_1/test_manager.py b/bbot/test/test_step_1/test_manager.py index 16e6db7f5..c863a600c 100644 --- a/bbot/test/test_step_1/test_manager.py +++ b/bbot/test/test_step_1/test_manager.py @@ -49,7 +49,7 @@ class DummyModule3: localhost.module = DummyModule1() # make sure abort_if works as intended await manager._emit_event(localhost, abort_if=lambda e: e.module._type == "output") - assert len(results) == 0 + assert not results manager.events_accepted.clear() manager.events_distributed.clear() await manager._emit_event(localhost, abort_if=lambda e: e.module._type != "output") @@ -67,7 +67,7 @@ class DummyModule3: # make sure deduplication is working localhost2 = scan1.make_event("127.0.0.1", source=scan1.root_event, tags=["localhost2"]) manager._emit_event(localhost2) - assert len(results) == 0 + assert not results # make sure dns resolution is working googledns = scan1.make_event("8.8.8.8", source=scan1.root_event) googledns.module = DummyModule2() @@ -75,7 +75,7 @@ class DummyModule3: googledns.set_scope_distance(0) manager.queue_event = event_children_append await manager._emit_event(googledns) - assert len(event_children) > 0 + assert event_children assert googledns in results assert googledns in output results.clear() @@ -83,16 +83,16 @@ class DummyModule3: event_children.clear() # make sure deduplication catches the same event await manager._emit_event(googledns) - assert len(output) == 0 - assert len(results) == 0 - assert len(event_children) == 0 + assert not output + assert not results + assert not event_children output.clear() event_children.clear() # make sure _force_output overrides dup detection googledns._force_output = True await manager._emit_event(googledns) assert googledns in output - assert len(event_children) == 0 + assert not event_children googledns._force_output = False results.clear() event_children.clear() @@ -101,7 +101,7 @@ class DummyModule3: source_event._resolved.set() googledns.source = source_event await manager._emit_event(googledns) - assert len(event_children) == 0 + assert not event_children assert googledns in output # error catching @@ -176,7 +176,7 @@ async def test_scope_distance(bbot_scanner, bbot_config): await manager._emit_event(test_event4) assert test_event4.scope_distance == 2 assert test_event4._internal == True - assert test_event4._force_output == True + assert test_event4._force_output assert test_event4 in output_queue assert test_event4 in module_queue valid, reason = await module._event_postcheck(test_event4) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index b4d61f516..6553f875d 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -122,8 +122,11 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h for flag in flags: all_flags.add(flag) if preloaded["type"] == "scan": - assert ("active" in flags and not "passive" in flags) or ( - not "active" in flags and "passive" in flags + assert ( + "active" in flags + and "passive" not in flags + or "active" not in flags + and "passive" in flags ), f'module "{module_name}" must have either "active" or "passive" flag' assert preloaded.get("meta", {}).get("description", ""), f"{module_name} must have a description" @@ -133,20 +136,20 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h assert type(watched_events) == list assert type(produced_events) == list - if not preloaded.get("type", "") in ("internal",): + if preloaded.get("type", "") not in ("internal",): assert watched_events, f"{module_name}.watched_events must not be empty" assert type(watched_events) == list, f"{module_name}.watched_events must be of type list" assert type(produced_events) == list, f"{module_name}.produced_events must be of type list" assert all( - [type(t) == str for t in watched_events] + type(t) == str for t in watched_events ), f"{module_name}.watched_events entries must be of type string" assert all( - [type(t) == str for t in produced_events] + type(t) == str for t in produced_events ), f"{module_name}.produced_events entries must be of type string" assert type(preloaded.get("deps_pip", [])) == list, f"{module_name}.deps_pip must be of type list" assert ( - type(preloaded.get("deps_pip_constraints", [])) == list + type(preloaded.get("deps_pip_constraints", [])) == list ), f"{module_name}.deps_pip_constraints must be of type list" assert type(preloaded.get("deps_apt", [])) == list, f"{module_name}.deps_apt must be of type list" assert type(preloaded.get("deps_shell", [])) == list, f"{module_name}.deps_shell must be of type list" @@ -158,7 +161,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h ), f"{module_name}.options do not match options_desc" # descriptions most not be blank assert all( - o for o in preloaded.get("options_desc", {}).values() + preloaded.get("options_desc", {}).values() ), f"{module_name}.options_desc descriptions must not be blank" from bbot.core.flags import flag_descriptions @@ -199,15 +202,15 @@ async def test_modules_basic_perhostonly(scan, helpers, events, bbot_config, bbo valid_2, reason_2 = await module._event_postcheck(url_2) if module.per_host_only == True: - assert valid_1 == True assert valid_2 == False assert hash("http://evilcorp.com/") in module._per_host_tracker assert reason_2 == "per_host_only enabled and already seen host" else: - assert valid_1 == True assert valid_2 == True + assert valid_1 == True + @pytest.mark.asyncio async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, bbot_scanner, httpx_mock, monkeypatch): @@ -240,11 +243,11 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, b valid_2, reason_2 = await module._event_postcheck(url_2) if module.per_domain_only == True: - assert valid_1 == True assert valid_2 == False assert hash("evilcorp.com") in module._per_host_tracker assert reason_2 == "per_domain_only enabled and already seen domain" else: - assert valid_1 == True assert valid_2 == True + + assert valid_1 == True diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 00ad2d972..b2c24f1f7 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -10,7 +10,7 @@ async def test_python_api(bbot_config): events1 = [] async for event in scan1.async_start(): events1.append(event) - assert any("127.0.0.1" == e for e in events1) + assert "127.0.0.1" in events1 # make sure output files work scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") await scan2.async_start_without_generator() @@ -51,10 +51,8 @@ def test_python_api_sync(bbot_config): # make sure events are properly yielded scan1 = Scanner("127.0.0.1", config=bbot_config) - events1 = [] - for event in scan1.start(): - events1.append(event) - assert any("127.0.0.1" == e for e in events1) + events1 = list(scan1.start()) + assert "127.0.0.1" in events1 # make sure output files work scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") scan2.start_without_generator() diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py index 7807e6c79..6d48ba678 100644 --- a/bbot/test/test_step_1/test_regexes.py +++ b/bbot/test/test_step_1/test_regexes.py @@ -42,12 +42,12 @@ def test_dns_name_regexes(): try: event_type, _ = get_event_type(dns) - if event_type == "OPEN_TCP_PORT": - assert dns == "evilcorp.com:80" - continue - elif event_type == "IP_ADDRESS": + if event_type == "IP_ADDRESS": assert dns == "1.2.3.4" continue + elif event_type == "OPEN_TCP_PORT": + assert dns == "evilcorp.com:80" + continue pytest.fail(f"BAD DNS NAME: {dns} matched returned event type: {event_type}") except ValidationError: continue @@ -55,12 +55,12 @@ def test_dns_name_regexes(): pytest.fail(f"BAD DNS NAME: {dns} raised unknown error: {e}") for dns in good_dns: - matches = list(r.match(dns) for r in dns_name_regexes) + matches = [r.match(dns) for r in dns_name_regexes] assert any(matches), f"Good DNS_NAME {dns} did not match regexes" event_type, _ = get_event_type(dns) - if not event_type == "DNS_NAME": + if event_type != "DNS_NAME": assert ( - dns == "1.2.3.4" and event_type == "IP_ADDRESS" + dns == "1.2.3.4" and event_type == "IP_ADDRESS" ), f"Event type for DNS_NAME {dns} was not properly detected" @@ -104,12 +104,12 @@ def test_open_port_regexes(): try: event_type, _ = get_event_type(open_port) - if event_type == "IP_ADDRESS": - assert open_port in ("1.2.3.4", "[dead::beef]") - continue - elif event_type == "DNS_NAME": + if event_type == "DNS_NAME": assert open_port in ("evilcorp.com", "asdfasdfasdfasdfasdfasdf.asdfasdfasdfasdfasdf.evilcorp.com") continue + elif event_type == "IP_ADDRESS": + assert open_port in ("1.2.3.4", "[dead::beef]") + continue pytest.fail(f"BAD OPEN_TCP_PORT: {open_port} matched returned event type: {event_type}") except ValidationError: continue @@ -117,7 +117,7 @@ def test_open_port_regexes(): pytest.fail(f"BAD OPEN_TCP_PORT: {open_port} raised unknown error: {e}") for open_port in good_ports: - matches = list(r.match(open_port) for r in open_port_regexes) + matches = [r.match(open_port) for r in open_port_regexes] assert any(matches), f"Good OPEN_TCP_PORT {open_port} did not match regexes" event_type, _ = get_event_type(open_port) assert event_type == "OPEN_TCP_PORT" @@ -182,8 +182,8 @@ def test_url_regexes(): pytest.fail(f"BAD URL: {bad_url} raised unknown error: {e}: {traceback.format_exc()}") for good_url in good_urls: - matches = list(r.match(good_url) for r in url_regexes) + matches = [r.match(good_url) for r in url_regexes] assert any(matches), f"Good URL {good_url} did not match regexes" assert ( - get_event_type(good_url)[0] == "URL_UNVERIFIED" + get_event_type(good_url)[0] == "URL_UNVERIFIED" ), f"Event type for URL {good_url} was not properly detected" diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index aa8140f5e..36ff6fefa 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -3,12 +3,12 @@ @pytest.mark.asyncio async def test_scan( - events, - bbot_config, - helpers, - neograph, - monkeypatch, - bbot_scanner, + events, + bbot_config, + helpers, + neograph, + monkeypatch, + bbot_scanner, ): scan0 = bbot_scanner( "8.8.8.8/31", diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index e51fec973..afa27096f 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -14,7 +14,7 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert not any(e.type == "URL" for e in events) + assert all(e.type != "URL" for e in events) class Scope_test_whitelist(Scope_test_blacklist): diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 90db526c7..fc9990a26 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -10,17 +10,18 @@ def test_target(bbot_config, bbot_scanner): assert not scan5.target assert len(scan1.target) == 9 assert len(scan4.target) == 8 - assert "8.8.8.9" in scan1.target - assert "8.8.8.12" not in scan1.target - assert "8.8.8.8/31" in scan1.target - assert "8.8.8.8/30" in scan1.target - assert "8.8.8.8/29" not in scan1.target - assert "2001:4860:4860::8889" in scan1.target + assert_target_presence("8.8.8.9", scan1, "8.8.8.12", "8.8.8.8/31") + assert_target_presence( + "8.8.8.8/30", scan1, "8.8.8.8/29", "2001:4860:4860::8889" + ) assert "2001:4860:4860::888c" not in scan1.target assert "www.api.publicapis.org" in scan1.target - assert "api.publicapis.org" in scan1.target - assert "publicapis.org" not in scan1.target - assert "bob@www.api.publicapis.org" in scan1.target + assert_target_presence( + "api.publicapis.org", + scan1, + "publicapis.org", + "bob@www.api.publicapis.org", + ) assert "https://www.api.publicapis.org" in scan1.target assert "www.api.publicapis.org:80" in scan1.target assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target @@ -38,3 +39,9 @@ def test_target(bbot_config, bbot_scanner): assert scan1.target.get("2001:4860:4860::888c") is None assert str(scan1.target.get("www.api.publicapis.org").host) == "api.publicapis.org" assert scan1.target.get("publicapis.org") is None + + +def assert_target_presence(arg0, scan1, arg2, arg3): + assert arg0 in scan1.target + assert arg2 not in scan1.target + assert arg3 in scan1.target diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index 334410f27..78438bd22 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -12,7 +12,7 @@ async def test_web_helpers(bbot_scanner, bbot_config, bbot_httpserver): user_agent = bbot_config.get("user_agent", "") headers = {"User-Agent": user_agent} custom_headers = bbot_config.get("http_headers", {}) - headers.update(custom_headers) + headers |= custom_headers assert headers["test"] == "header" url = bbot_httpserver.url_for("/test_http_helpers") @@ -135,17 +135,17 @@ async def test_web_curl(bbot_scanner, bbot_config, bbot_httpserver): assert (await helpers.curl(url=url, head_mode=True)).startswith("HTTP/") assert await helpers.curl(url=url, raw_body="body") == "curl_yep" assert ( - await helpers.curl( - url=url, - raw_path=True, - headers={"test": "test", "test2": ["test2"]}, - ignore_bbot_global_settings=False, - post_data={"test": "test"}, - method="POST", - cookies={"test": "test"}, - path_override="/index.html", - ) - == "curl_yep_index" + await helpers.curl( + url=url, + raw_path=True, + headers={"test": "test", "test2": ["test2"]}, + ignore_bbot_global_settings=False, + post_data={"test": "test"}, + method="POST", + cookies={"test": "test"}, + path_override="/index.html", + ) + == "curl_yep_index" ) # test custom headers bbot_httpserver.expect_request("/test-custom-http-headers-curl", headers={"test": "header"}).respond_with_data( @@ -186,7 +186,7 @@ async def test_http_proxy(bbot_scanner, bbot_config, bbot_httpserver, proxy_serv r = await scan.helpers.request(url) assert ( - len(proxy_server.RequestHandlerClass.urls) == 1 + len(proxy_server.RequestHandlerClass.urls) == 1 ), f"Request to {url} did not go through proxy {proxy_address}" visited_url = proxy_server.RequestHandlerClass.urls[0] assert visited_url.endswith(endpoint), f"There was a problem with request to {url}: {visited_url}" diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index a4562cfc7..99f86c09b 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -85,7 +85,11 @@ def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, mon self.events = [] self.log = logging.getLogger(f"bbot.test.{module_test_base.name}") - def set_expect_requests(self, expect_args={}, respond_args={}): + def set_expect_requests(self, expect_args=None, respond_args=None): + if expect_args is None: + expect_args = {} + if respond_args is None: + respond_args = {} if "uri" not in expect_args: expect_args["uri"] = "/" self.httpserver.expect_request(**expect_args).respond_with_data(**respond_args) @@ -115,8 +119,7 @@ async def test_module_run(self, module_test): self.check(module_test, module_test.events) module_test.log.info(f"Finished {self.name} module test") current_task = asyncio.current_task() - tasks = [t for t in asyncio.all_tasks() if t != current_task] - if len(tasks) > 0: + if tasks := [t for t in asyncio.all_tasks() if t != current_task]: module_test.log.info(f"Unfinished tasks detected: {tasks}") def check(self, module_test, events): @@ -138,9 +141,7 @@ def _scan_name(self): @property def modules(self): - if self.modules_overrides: - return self.modules_overrides - return [self.name] + return self.modules_overrides or [self.name] async def setup_before_prep(self, module_test): pass diff --git a/bbot/test/test_step_2/module_tests/test_module_anubisdb.py b/bbot/test/test_step_2/module_tests/test_module_anubisdb.py index dbebf8621..d6e99fc0c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_anubisdb.py +++ b/bbot/test/test_step_2/module_tests/test_module_anubisdb.py @@ -5,8 +5,11 @@ class TestAnubisdb(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( - url=f"https://jldc.me/anubis/subdomains/blacklanternsecurity.com", - json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], + url="https://jldc.me/anubis/subdomains/blacklanternsecurity.com", + json=[ + "asdf.blacklanternsecurity.com", + "zzzz.blacklanternsecurity.com", + ], ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_azure_realm.py b/bbot/test/test_step_2/module_tests/test_module_azure_realm.py index 7ab5463c1..3e4bc699c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_azure_realm.py +++ b/bbot/test/test_step_2/module_tests/test_module_azure_realm.py @@ -21,7 +21,7 @@ class TestAzure_Realm(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com", + url="https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com", json=self.response_json, ) diff --git a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py index 843e1461f..8571894ee 100644 --- a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py +++ b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py @@ -77,33 +77,33 @@ def check(self, module_test, events): for e in events: if ( - e.type == "VULNERABILITY" - and "Known Secret Found." in e.data["description"] - and "validationKey: 0F97BAE23F6F36801ABDB5F145124E00A6F795A97093D778EE5CD24F35B78B6FC4C0D0D4420657689C4F321F8596B59E83F02E296E970C4DEAD2DFE226294979 validationAlgo: SHA1 encryptionKey: 8CCFBC5B7589DD37DC3B4A885376D7480A69645DAEEC74F418B4877BEC008156 encryptionAlgo: AES" - in e.data["description"] + e.type == "VULNERABILITY" + and "Known Secret Found." in e.data["description"] + and "validationKey: 0F97BAE23F6F36801ABDB5F145124E00A6F795A97093D778EE5CD24F35B78B6FC4C0D0D4420657689C4F321F8596B59E83F02E296E970C4DEAD2DFE226294979 validationAlgo: SHA1 encryptionKey: 8CCFBC5B7589DD37DC3B4A885376D7480A69645DAEEC74F418B4877BEC008156 encryptionAlgo: AES" + in e.data["description"] ): SecretFound = True if ( - e.type == "FINDING" - and "AAAAYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESfAAAA" - in e.data["description"] + e.type == "FINDING" + and "AAAAYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESfAAAA" + in e.data["description"] ): IdentifyOnly = True if ( - e.type == "VULNERABILITY" - and "1234" in e.data["description"] - and "eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo" - in e.data["description"] + e.type == "VULNERABILITY" + and "1234" in e.data["description"] + and "eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo" + in e.data["description"] ): CookieBasedDetection = True if ( - e.type == "VULNERABILITY" - and "keyboard cat" in e.data["description"] - and "s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc" - in e.data["description"] + e.type == "VULNERABILITY" + and "keyboard cat" in e.data["description"] + and "s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc" + in e.data["description"] ): CookieBasedDetection_2 = True diff --git a/bbot/test/test_step_2/module_tests/test_module_bevigil.py b/bbot/test/test_step_2/module_tests/test_module_bevigil.py index e8ab13d7a..72ff1eb08 100644 --- a/bbot/test/test_step_2/module_tests/test_module_bevigil.py +++ b/bbot/test/test_step_2/module_tests/test_module_bevigil.py @@ -6,7 +6,7 @@ class TestBeVigil(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/", + url="https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/", json={ "domain": "blacklanternsecurity.com", "subdomains": [ @@ -15,8 +15,11 @@ async def setup_after_prep(self, module_test): }, ) module_test.httpx_mock.add_response( - url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/urls/", - json={"domain": "blacklanternsecurity.com", "urls": ["https://asdf.blacklanternsecurity.com"]}, + url="https://osint.bevigil.com/api/blacklanternsecurity.com/urls/", + json={ + "domain": "blacklanternsecurity.com", + "urls": ["https://asdf.blacklanternsecurity.com"], + }, ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_binaryedge.py b/bbot/test/test_step_2/module_tests/test_module_binaryedge.py index 505c62376..9e55e81e8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_binaryedge.py +++ b/bbot/test/test_step_2/module_tests/test_module_binaryedge.py @@ -6,7 +6,7 @@ class TestBinaryEdge(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.binaryedge.io/v2/query/domains/subdomain/blacklanternsecurity.com", + url="https://api.binaryedge.io/v2/query/domains/subdomain/blacklanternsecurity.com", json={ "query": "blacklanternsecurity.com", "page": 1, @@ -18,7 +18,7 @@ async def setup_before_prep(self, module_test): }, ) module_test.httpx_mock.add_response( - url=f"https://api.binaryedge.io/v2/user/subscription", + url="https://api.binaryedge.io/v2/user/subscription", json={ "subscription": {"name": "Free"}, "end_date": "2023-06-17", diff --git a/bbot/test/test_step_2/module_tests/test_module_bucket_aws.py b/bbot/test/test_step_2/module_tests/test_module_bucket_aws.py index d11cc304b..c1629ce95 100644 --- a/bbot/test/test_step_2/module_tests/test_module_bucket_aws.py +++ b/bbot/test/test_step_2/module_tests/test_module_bucket_aws.py @@ -78,8 +78,8 @@ def check(self, module_test, events): if e.type == "FINDING" and str(e.module) == f"bucket_{self.provider}": url = e.data.get("url", "") assert self.random_bucket_2 in url - assert not self.random_bucket_1 in url - assert not f"{self.random_bucket_3}" in url + assert self.random_bucket_1 not in url + assert f"{self.random_bucket_3}" not in url # make sure bucket mutations were found assert any( e.type == "STORAGE_BUCKET" diff --git a/bbot/test/test_step_2/module_tests/test_module_builtwith.py b/bbot/test/test_step_2/module_tests/test_module_builtwith.py index 0fc4de9d5..d11c8940d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_builtwith.py +++ b/bbot/test/test_step_2/module_tests/test_module_builtwith.py @@ -6,7 +6,7 @@ class TestBuiltWith(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.builtwith.com/v20/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes", + url="https://api.builtwith.com/v20/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes", json={ "Results": [ { @@ -91,7 +91,7 @@ async def setup_after_prep(self, module_test): }, ) module_test.httpx_mock.add_response( - url=f"https://api.builtwith.com/redirect1/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com", + url="https://api.builtwith.com/redirect1/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com", json={ "Lookup": "blacklanternsecurity.com", "Inbound": [ diff --git a/bbot/test/test_step_2/module_tests/test_module_certspotter.py b/bbot/test/test_step_2/module_tests/test_module_certspotter.py index a9ab7eb36..a377e8b5b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_certspotter.py +++ b/bbot/test/test_step_2/module_tests/test_module_certspotter.py @@ -4,7 +4,7 @@ class TestCertspotter(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False - for t in self.targets: + for _ in self.targets: module_test.httpx_mock.add_response( url="https://api.certspotter.com/v1/issuances?domain=blacklanternsecurity.com&include_subdomains=true&expand=dns_names", json=[{"dns_names": ["*.asdf.blacklanternsecurity.com"]}], diff --git a/bbot/test/test_step_2/module_tests/test_module_columbus.py b/bbot/test/test_step_2/module_tests/test_module_columbus.py index 55d456ce3..b91b532d7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_columbus.py +++ b/bbot/test/test_step_2/module_tests/test_module_columbus.py @@ -4,7 +4,7 @@ class TestColumbus(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://columbus.elmasy.com/api/lookup/blacklanternsecurity.com?days=365", + url="https://columbus.elmasy.com/api/lookup/blacklanternsecurity.com?days=365", json=["asdf", "zzzz"], ) diff --git a/bbot/test/test_step_2/module_tests/test_module_credshed.py b/bbot/test/test_step_2/module_tests/test_module_credshed.py index 7de642412..400266e18 100644 --- a/bbot/test/test_step_2/module_tests/test_module_credshed.py +++ b/bbot/test/test_step_2/module_tests/test_module_credshed.py @@ -1,12 +1,10 @@ from .base import ModuleTestBase - credshed_auth_response = { "access_token": "big_access_token", "login": True, } - credshed_response = { "accounts": [ { @@ -58,12 +56,12 @@ class TestCredshed(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://credshed.com/api/auth", + url="https://credshed.com/api/auth", json=credshed_auth_response, method="POST", ) module_test.httpx_mock.add_response( - url=f"https://credshed.com/api/search", + url="https://credshed.com/api/search", json=credshed_response, method="POST", ) @@ -84,7 +82,7 @@ def check(self, module_test, events): e for e in events if e.type == "HASHED_PASSWORD" - and e.data == "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm" + and e.data == "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm" ] ) assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_crt.py b/bbot/test/test_step_2/module_tests/test_module_crt.py index 5ee8ae4d3..89ac6fcea 100644 --- a/bbot/test/test_step_2/module_tests/test_module_crt.py +++ b/bbot/test/test_step_2/module_tests/test_module_crt.py @@ -4,7 +4,7 @@ class TestCRT(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False - for t in self.targets: + for _ in self.targets: module_test.httpx_mock.add_response( url="https://crt.sh?q=%25.blacklanternsecurity.com&output=json", json=[{"id": 1, "name_value": "asdf.blacklanternsecurity.com\nzzzz.blacklanternsecurity.com"}], diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py index 767884bd5..dba1a3ff9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -41,7 +41,7 @@ class TestDehashed(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.dehashed.com/search?query=domain:blacklanternsecurity.com&size=10000&page=1", + url="https://api.dehashed.com/search?query=domain:blacklanternsecurity.com&size=10000&page=1", json=dehashed_domain_response, ) @@ -54,7 +54,7 @@ def check(self, module_test, events): e for e in events if e.type == "HASHED_PASSWORD" - and e.data == "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve" + and e.data == "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve" ] ) assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_digitorus.py b/bbot/test/test_step_2/module_tests/test_module_digitorus.py index fc95a82c7..a683a17d8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_digitorus.py +++ b/bbot/test/test_step_2/module_tests/test_module_digitorus.py @@ -11,7 +11,7 @@ class TestDigitorus(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://certificatedetails.com/blacklanternsecurity.com", + url="https://certificatedetails.com/blacklanternsecurity.com", text=self.web_response, ) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index bdb6f1207..901dc0eb2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -9,8 +9,8 @@ async def setup_after_prep(self, module_test): async def resolve(query, **kwargs): if ( - query == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" - and kwargs.get("type", "").upper() == "SRV" + query == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" + and kwargs.get("type", "").upper() == "SRV" ): return {"asdf.blacklanternsecurity.notreal"} return await old_resolve_fn(query, **kwargs) @@ -21,4 +21,7 @@ def check(self, module_test, events): assert any( e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" for e in events ), "Failed to detect subdomain" - assert not any(e.data == "_ldap._tcp.dc._msdcs.blacklanternsecurity.notreal" for e in events), "False positive" + assert all( + e.data != "_ldap._tcp.dc._msdcs.blacklanternsecurity.notreal" + for e in events + ), "False positive" diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py b/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py index 6bf045d5c..447f8baba 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py @@ -4,12 +4,12 @@ class TestDNSDumpster(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://dnsdumpster.com", + url="https://dnsdumpster.com", headers={"Set-Cookie": "csrftoken=asdf"}, content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n
\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similiar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n
\n\n\n
\n
\n
\n
\n

About

\n

At hackertarget.com, we have been scanning the Internet since 2007. Our vulnerability scanners now reach millions of IP addresses every year. We work to raise awareness of the value in open source security solutions.

\n
\n
\n
\n
\n

Stay in Touch

\n\n
\n

dnsdumpster@gmail.com

\n
\n
\n

@hackertarget

\n
\n\n\n\n
\n
\n
\n\n \n \n
\n
Low volume Updates and News
\n
\n
\n
\n\n \n\n
\n
\n
\n
\n
\n

Copyright © 2023 Hacker Target Pty Ltd

\n
\n
\n\n\n
\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n\n\n\n\n\n\n \n\n', ) module_test.httpx_mock.add_response( - url=f"https://dnsdumpster.com/", + url="https://dnsdumpster.com/", method="POST", content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n
\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n

Showing results for blacklanternsecurity.com

\n
\n
\n
\n
\n

Hosting (IP block owners)

\n
\n
\n

GeoIP of Host Locations

\n
\n
\n
\n\n

DNS Servers

\n
\n \n \n \n \n \n \n
ns01.domaincontrol.com.
\n\n\n \n
\n
\n		97.74.100.1
ns01.domaincontrol.com		GODADDY-DNS
United States
ns02.domaincontrol.com.
\n\n\n \n
\n
\n		173.201.68.1
ns02.domaincontrol.com		GODADDY-DNS
United States
\n
\n\n

MX Records ** This is where email for the domain goes...

\n
\n \n \n \n \n
asdf.blacklanternsecurity.com.mail.protection.outlook.com.
\n\n\n
\n
\n		104.47.55.138
mail-bn8nam120138.inbound.protection.outlook.com		MICROSOFT-CORP-MSN-AS-BLOCK
United States
\n
\n\n

TXT Records ** Find more hosts in Sender Policy Framework (SPF) configurations

\n
\n \n\n\n\n\n\n\n\n\n\n
"MS=ms26206678"
"v=spf1 ip4:50.240.76.25 include:spf.protection.outlook.com -all"
"google-site-verification=O_PoQFTGJ_hZ9LqfNT9OEc0KPFERKHQ_1t1m0YTx_1E"
"google-site-verification=7XKUMxJSTHBSzdvT7gH47jLRjNAS76nrEfXmzhR_DO4"
\n
\n\n\n

Host Records (A) ** this data may not be current as it uses a static database (updated monthly)

\n
\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n GitHub.com\n\n\n\n\n\n\n\n\n
HTTP TECH: \n varnish\n\n\n\n		185.199.108.153
cdn-185-199-108-153.github.com		FASTLY
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n
SSH: \n SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.3\n\n\n\n\n\n\n\n		143.244.156.80
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n Apache/2.4.29 (Ubuntu)\n\n\n\n\n\n\n\n\n
HTTP TECH: \n Ubuntu
Apache,2.4.29\n\n\n\n		64.227.8.231
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		192.34.56.157
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		192.241.216.208
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		167.71.95.71
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		157.245.247.197
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
\n
\n\n\n\n\n\n
\n

Mapping the domain ** click for full size image

\n

\n\n

\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similiar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n\n\n\n\n
\n\n\n
\n
\n
\n
\n

About

\n

At hackertarget.com, we have been scanning the Internet since 2007. Our vulnerability scanners now reach millions of IP addresses every year. We work to raise awareness of the value in open source security solutions.

\n
\n
\n
\n
\n

Stay in Touch

\n\n
\n

dnsdumpster@gmail.com

\n
\n
\n

@hackertarget

\n
\n\n\n\n
\n
\n
\n\n \n \n
\n
Low volume Updates and News
\n
\n
\n
\n\n \n\n
\n
\n
\n
\n
\n

Copyright © 2023 Hacker Target Pty Ltd

\n
\n
\n\n\n
\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n \n \n \n\n \n\n\n\n\n\n\n\n\n\n\n\n\n \n\n', ) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 18d453b03..f5394b93e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -129,17 +129,16 @@ def check(self, module_test, events): for e in events: if e.type == "URL_UNVERIFIED": # these cases represent the desired behavior for parsing relative links - if e.data == "http://127.0.0.1:8888/rootrelative.html": + if e.data == "http://127.0.0.1:8888/pagerelative.html": + root_page_confusion_1 = True + elif e.data == "http://127.0.0.1:8888/rootrelative.html": root_relative_detection = True - if e.data == "http://127.0.0.1:8888/subdir/pagerelative1.html": + elif e.data == "http://127.0.0.1:8888/subdir/pagerelative1.html": page_relative_detection_1 = True - if e.data == "http://127.0.0.1:8888/subdir/pagerelative2.html": + elif e.data == "http://127.0.0.1:8888/subdir/pagerelative2.html": page_relative_detection_2 = True - # these cases indicates that excavate parsed the relative links incorrectly - if e.data == "http://127.0.0.1:8888/pagerelative.html": - root_page_confusion_1 = True - if e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": + elif e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": root_page_confusion_2 = True assert root_relative_detection, "Failed to properly excavate root-relative URL" diff --git a/bbot/test/test_step_2/module_tests/test_module_filedownload.py b/bbot/test/test_step_2/module_tests/test_module_filedownload.py index e4471d159..8b14a0c52 100644 --- a/bbot/test/test_step_2/module_tests/test_module_filedownload.py +++ b/bbot/test/test_step_2/module_tests/test_module_filedownload.py @@ -49,20 +49,24 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): download_dir = module_test.scan.home / "filedownload" - # text file - text_files = list(download_dir.glob("*test-file.txt")) - assert len(text_files) == 1, f"No text file found at {download_dir}" - file = text_files[0] - assert file.is_file(), f"File not found at {file}" + file = self.assert_single_file_presence( + download_dir, "*test-file.txt", 'No text file found at ' + ) assert open(file).read() == "juicy stuff", f"File at {file} does not contain the correct content" - # PDF file (no extension) - pdf_files = list(download_dir.glob("*test-pdf.pdf")) - assert len(pdf_files) == 1, f"No PDF file found at {download_dir}" - file = pdf_files[0] - assert file.is_file(), f"File not found at {file}" + file = self.assert_single_file_presence( + download_dir, "*test-pdf.pdf", 'No PDF file found at ' + ) assert open(file).read() == self.pdf_data, f"File at {file} does not contain the correct content" # we don't want html files html_files = list(download_dir.glob("*.html")) - assert len(html_files) == 0, "HTML files were erroneously downloaded" + assert not html_files, "HTML files were erroneously downloaded" + + def assert_single_file_presence(self, download_dir, arg1, arg2): + # text file + text_files = list(download_dir.glob(arg1)) + assert len(text_files) == 1, f"{arg2}{download_dir}" + result = text_files[0] + assert result.is_file(), f"File not found at {result}" + return result diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py index 370dd151a..a6e14f902 100644 --- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py @@ -6,9 +6,8 @@ def extract_subdomain_tag(data): pattern = r"http://([a-z0-9]{4})\.fakedomain\.fakeinteractsh\.com" - match = re.search(pattern, data) - if match: - return match.group(1) + if match := re.search(pattern, data): + return match[1] class TestGeneric_SSRF(ModuleTestBase): diff --git a/bbot/test/test_step_2/module_tests/test_module_host_header.py b/bbot/test/test_step_2/module_tests/test_module_host_header.py index 9741d9bc1..089f6ee85 100644 --- a/bbot/test/test_step_2/module_tests/test_module_host_header.py +++ b/bbot/test/test_step_2/module_tests/test_module_host_header.py @@ -6,8 +6,7 @@ def extract_subdomain_tag(data): pattern = r"([a-z0-9]{4})\.fakedomain\.fakeinteractsh\.com" - match = re.search(pattern, data) - if match: + if match := re.search(pattern, data): return match.group(1) @@ -26,12 +25,12 @@ def request_handler(self, request): self.interactsh_mock_instance.mock_interaction(subdomain_tag) return Response(f"Alive, host is: {subdomain_tag}.{self.fake_host}", status=200) - # Host Header Overrides - subdomain_tag_overrides = extract_subdomain_tag(request.headers["X-Forwarded-For"]) - if subdomain_tag_overrides: + if subdomain_tag_overrides := extract_subdomain_tag( + request.headers["X-Forwarded-For"] + ): return Response(f"Alive, host is: {subdomain_tag}.{self.fake_host}", status=200) - return Response(f"Alive, host is: defaulthost.com", status=200) + return Response("Alive, host is: defaulthost.com", status=200) async def setup_before_prep(self, module_test): self.interactsh_mock_instance = module_test.request_fixture.getfixturevalue("interactsh_mock_instance") diff --git a/bbot/test/test_step_2/module_tests/test_module_httpx.py b/bbot/test/test_step_2/module_tests/test_module_httpx.py index fcf134dd3..76247df42 100644 --- a/bbot/test/test_step_2/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_2/module_tests/test_module_httpx.py @@ -41,7 +41,7 @@ def check(self, module_test, events): for e in events: if e.type == "HTTP_RESPONSE": if e.data["path"] == "/": - assert not "login-page" in e.tags + assert "login-page" not in e.tags open_port = True elif e.data["path"] == "/url": assert "login-page" in e.tags diff --git a/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py b/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py index fbccc69a7..2591cdbf3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py +++ b/bbot/test/test_step_2/module_tests/test_module_ipneighbor.py @@ -24,4 +24,4 @@ async def _resolve_hostname(query, **kwargs): def check(self, module_test, events): assert any(e.data == "127.0.0.3" for e in events) - assert not any(e.data == "127.0.0.4" for e in events) + assert all(e.data != "127.0.0.4" for e in events) diff --git a/bbot/test/test_step_2/module_tests/test_module_leakix.py b/bbot/test/test_step_2/module_tests/test_module_leakix.py index aad4a095c..3267df93a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_leakix.py +++ b/bbot/test/test_step_2/module_tests/test_module_leakix.py @@ -11,7 +11,7 @@ async def setup_before_prep(self, module_test): json={"title": "Not Found", "description": "Host not found"}, ) module_test.httpx_mock.add_response( - url=f"https://leakix.net/api/subdomains/blacklanternsecurity.com", + url="https://leakix.net/api/subdomains/blacklanternsecurity.com", match_headers={"api-key": "asdf"}, json=[ { @@ -31,7 +31,7 @@ class TestLeakIX_NoAPIKey(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://leakix.net/api/subdomains/blacklanternsecurity.com", + url="https://leakix.net/api/subdomains/blacklanternsecurity.com", json=[ { "subdomain": "asdf.blacklanternsecurity.com", diff --git a/bbot/test/test_step_2/module_tests/test_module_massdns.py b/bbot/test/test_step_2/module_tests/test_module_massdns.py index 04f4860dd..e73abcaf0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_massdns.py +++ b/bbot/test/test_step_2/module_tests/test_module_massdns.py @@ -13,4 +13,4 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): assert any(e.data == "www.blacklanternsecurity.com" for e in events) - assert not any(e.data == "asdf.blacklanternsecurity.com" for e in events) + assert all(e.data != "asdf.blacklanternsecurity.com" for e in events) diff --git a/bbot/test/test_step_2/module_tests/test_module_myssl.py b/bbot/test/test_step_2/module_tests/test_module_myssl.py index 34b9b9972..b39f2711d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_myssl.py +++ b/bbot/test/test_step_2/module_tests/test_module_myssl.py @@ -5,7 +5,7 @@ class TestMySSL(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( - url=f"https://myssl.com/api/v1/discover_sub_domain?domain=blacklanternsecurity.com", + url="https://myssl.com/api/v1/discover_sub_domain?domain=blacklanternsecurity.com", json={ "code": 0, "data": [ diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py index 58bb801d1..a8e0454f5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_nmap.py +++ b/bbot/test/test_step_2/module_tests/test_module_nmap.py @@ -7,4 +7,4 @@ class TestNmap(ModuleTestBase): def check(self, module_test, events): assert any(e.data == "127.0.0.1:8888" for e in events) - assert not any(e.data == "127.0.0.1:8889" for e in events) + assert all(e.data != "127.0.0.1:8889" for e in events) diff --git a/bbot/test/test_step_2/module_tests/test_module_otx.py b/bbot/test/test_step_2/module_tests/test_module_otx.py index 1c41cd962..9c533ca96 100644 --- a/bbot/test/test_step_2/module_tests/test_module_otx.py +++ b/bbot/test/test_step_2/module_tests/test_module_otx.py @@ -4,7 +4,7 @@ class TestOTX(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://otx.alienvault.com/api/v1/indicators/domain/blacklanternsecurity.com/passive_dns", + url="https://otx.alienvault.com/api/v1/indicators/domain/blacklanternsecurity.com/passive_dns", json={ "passive_dns": [ { diff --git a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py index be49cb881..664303673 100644 --- a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py +++ b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py @@ -9,7 +9,8 @@ class TestRapidDNS(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( - url=f"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result", text=self.web_body + url="https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result", + text=self.web_body, ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_riddler.py b/bbot/test/test_step_2/module_tests/test_module_riddler.py index d7be9de56..3a52ea325 100644 --- a/bbot/test/test_step_2/module_tests/test_module_riddler.py +++ b/bbot/test/test_step_2/module_tests/test_module_riddler.py @@ -9,7 +9,8 @@ class TestRiddler(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( - url=f"https://riddler.io/search/exportcsv?q=pld:blacklanternsecurity.com", text=self.web_body + url="https://riddler.io/search/exportcsv?q=pld:blacklanternsecurity.com", + text=self.web_body, ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_robots.py b/bbot/test/test_step_2/module_tests/test_module_robots.py index 3d9156bb4..308447060 100644 --- a/bbot/test/test_step_2/module_tests/test_module_robots.py +++ b/bbot/test/test_step_2/module_tests/test_module_robots.py @@ -27,10 +27,10 @@ def check(self, module_test, events): if e.data == "http://127.0.0.1:8888/allow/": allow_bool = True - if e.data == "http://127.0.0.1:8888/disallow/": + elif e.data == "http://127.0.0.1:8888/disallow/": disallow_bool = True - if e.data == "http://127.0.0.1:8888/sitemap.txt": + elif e.data == "http://127.0.0.1:8888/sitemap.txt": sitemap_bool = True if re.match(r"http://127\.0\.0\.1:8888/\w+/wildcard\.txt", e.data): diff --git a/bbot/test/test_step_2/module_tests/test_module_sitedossier.py b/bbot/test/test_step_2/module_tests/test_module_sitedossier.py index 2156a5ae7..053c80086 100644 --- a/bbot/test/test_step_2/module_tests/test_module_sitedossier.py +++ b/bbot/test/test_step_2/module_tests/test_module_sitedossier.py @@ -127,11 +127,10 @@ class TestSitedossier(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"http://www.sitedossier.com/parentdomain/evilcorp.com", - text=page1, + url="http://www.sitedossier.com/parentdomain/evilcorp.com", text=page1 ) module_test.httpx_mock.add_response( - url=f"http://www.sitedossier.com/parentdomain/evilcorp.com/101", + url="http://www.sitedossier.com/parentdomain/evilcorp.com/101", text=page2, ) diff --git a/bbot/test/test_step_2/module_tests/test_module_smuggler.py b/bbot/test/test_step_2/module_tests/test_module_smuggler.py index c887d684a..4c7edf17a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_smuggler.py +++ b/bbot/test/test_step_2/module_tests/test_module_smuggler.py @@ -39,7 +39,7 @@ async def setup_after_prep(self, module_test): old_run_live = module_test.scan.helpers.run_live async def smuggler_mock_run_live(*command, **kwargs): - if not "smuggler" in command[0][1]: + if "smuggler" not in command[0][1]: async for l in old_run_live(*command, **kwargs): yield l else: diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py b/bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py index 2ec5e0361..c057b8e4e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py +++ b/bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py @@ -4,8 +4,11 @@ class TestSubdomainCenter(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.subdomain.center/?domain=blacklanternsecurity.com", - json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], + url="https://api.subdomain.center/?domain=blacklanternsecurity.com", + json=[ + "asdf.blacklanternsecurity.com", + "zzzz.blacklanternsecurity.com", + ], ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomains.py b/bbot/test/test_step_2/module_tests/test_module_subdomains.py index 9aa9f7b5e..0661f5e8e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_subdomains.py +++ b/bbot/test/test_step_2/module_tests/test_module_subdomains.py @@ -6,8 +6,11 @@ class TestSubdomains(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.subdomain.center/?domain=blacklanternsecurity.com", - json=["asdfasdf.blacklanternsecurity.com", "zzzzzzzz.blacklanternsecurity.com"], + url="https://api.subdomain.center/?domain=blacklanternsecurity.com", + json=[ + "asdfasdf.blacklanternsecurity.com", + "zzzzzzzz.blacklanternsecurity.com", + ], ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_sublist3r.py b/bbot/test/test_step_2/module_tests/test_module_sublist3r.py index 656b4696c..6484f09ac 100644 --- a/bbot/test/test_step_2/module_tests/test_module_sublist3r.py +++ b/bbot/test/test_step_2/module_tests/test_module_sublist3r.py @@ -4,8 +4,11 @@ class TestSublist3r(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"https://api.sublist3r.com/search.php?domain=blacklanternsecurity.com", - json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], + url="https://api.sublist3r.com/search.php?domain=blacklanternsecurity.com", + json=[ + "asdf.blacklanternsecurity.com", + "zzzz.blacklanternsecurity.com", + ], ) def check(self, module_test, events): diff --git a/bbot/test/test_step_2/module_tests/test_module_telerik.py b/bbot/test/test_step_2/module_tests/test_module_telerik.py index 6a4a7d97d..534911db0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_telerik.py +++ b/bbot/test/test_step_2/module_tests/test_module_telerik.py @@ -64,7 +64,7 @@ def check(self, module_test, events): telerik_axd_detection = True continue - if e.type == "VULNERABILITY" and "Confirmed Vulnerable Telerik (version: 2014.3.1024)": + if e.type == "VULNERABILITY": telerik_axd_vulnerable = True continue diff --git a/bbot/test/test_step_2/module_tests/test_module_wayback.py b/bbot/test/test_step_2/module_tests/test_module_wayback.py index cf09d8e2c..7582e5417 100644 --- a/bbot/test/test_step_2/module_tests/test_module_wayback.py +++ b/bbot/test/test_step_2/module_tests/test_module_wayback.py @@ -4,7 +4,7 @@ class TestWayback(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( - url=f"http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original", + url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original", json=[["original"], ["http://asdf.blacklanternsecurity.com"]], ) diff --git a/bbot/test/test_step_2/module_tests/test_module_web_report.py b/bbot/test/test_step_2/module_tests/test_module_web_report.py index aa51d501a..b278564a0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_web_report.py +++ b/bbot/test/test_step_2/module_tests/test_module_web_report.py @@ -1,3 +1,4 @@ +import pathlib from .base import ModuleTestBase @@ -22,20 +23,19 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): report_file = module_test.scan.home / "web_report.html" - with open(report_file) as f: - report_content = f.read() + report_content = pathlib.Path(report_file).read_text() assert "
  • [CRITICAL] Known Secret Found" in report_content assert ( - """

    URL

    -
      -
    • http://127.0.0.1:8888/""" - in report_content + """

      URL

      +
        +
      • http://127.0.0.1:8888/""" + in report_content ) assert ( - """

        FINDING

        -
          -
        • Possible secret (Asymmetric Private Key)""" - in report_content + """

          FINDING

          +
            +
          • Possible secret (Asymmetric Private Key)""" + in report_content ) assert "

            TECHNOLOGY

            " in report_content assert "

            flask

            " in report_content diff --git a/bbot/test/test_step_2/module_tests/test_module_zoomeye.py b/bbot/test/test_step_2/module_tests/test_module_zoomeye.py index c6c0b5328..130544444 100644 --- a/bbot/test/test_step_2/module_tests/test_module_zoomeye.py +++ b/bbot/test/test_step_2/module_tests/test_module_zoomeye.py @@ -32,4 +32,6 @@ def check(self, module_test, events): assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #2" assert any(e.data == "ffff.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #3" assert any(e.data == "affiliate.bls" and "affiliate" in e.tags for e in events), "Failed to detect affiliate" - assert not any(e.data == "nope.blacklanternsecurity.com" for e in events), "Failed to obey max_pages" + assert all( + e.data != "nope.blacklanternsecurity.com" for e in events + ), "Failed to obey max_pages"