From 8be8216521a68ed400e78134a1aca9a5c1de25db Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 May 2023 16:42:32 -0400 Subject: [PATCH 001/387] massdns improvements, added module sequence --- bbot/core/event/base.py | 14 ++++ bbot/modules/httpx.py | 4 +- bbot/modules/massdns.py | 140 +++++++++++++++++++---------------- bbot/modules/output/csv.py | 2 +- bbot/modules/output/human.py | 2 +- 5 files changed, 95 insertions(+), 67 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index cd16574dac..a88c5314fc 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -447,6 +447,8 @@ def json(self, mode="json"): j.update({"tags": list(self.tags)}) if self.module: j.update({"module": str(self.module)}) + if self.module_sequence: + j.update({"module_sequence": str(self.module_sequence)}) # normalize non-primitive python objects for k, v in list(j.items()): @@ -463,6 +465,18 @@ def json(self, mode="json"): def from_json(j): return event_from_json(j) + @property + def module_sequence(self): + """ + A human-friendly representation of the module name that includes modules from omitted source events + + Helpful in identifying where a URL came from + """ + module_name = getattr(self.module, "name", "") + if getattr(self.source, "_omit", False): + module_name = f"{self.source.module_sequence}->{module_name}" + return module_name + @property def module_priority(self): if self._module_priority is None: diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index d367951ac9..6641b38301 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -136,14 +136,14 @@ def handle_batch(self, *events): title = self.helpers.tagify(j.get("title", "")) if title: tags.append(f"http-title-{title}") - url_event = self.make_event(url, "URL", source_event, module=source_event.module, tags=tags) + url_event = self.make_event(url, "URL", source_event, tags=tags) if url_event: if url_event != source_event: self.emit_event(url_event) else: url_event._resolved.set() # HTTP response - self.emit_event(j, "HTTP_RESPONSE", url_event, module=source_event.module, internal=True) + self.emit_event(j, "HTTP_RESPONSE", url_event, internal=True) def cleanup(self): resume_file = self.helpers.current_dir / "resume.cfg" diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index fd8b58bc85..10b69aa0cd 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -233,80 +233,94 @@ def _massdns(self, domain, subdomains): def finish(self): found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True) + # if we have a lot of rounds to make, don't try mutations on less-populated domains + avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50]) + trimmed_found = [] + for i, (domain, subdomains) in enumerate(found): + # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains + if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)): + trimmed_found.append((domain, subdomains)) + else: + self.verbose( + f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})" + ) base_mutations = set() - for i, (domain, subdomains) in enumerate(found): - # keep looping as long as we're finding things - while 1: - max_mem_percent = 90 - mem_status = self.helpers.memory_status() - # abort if we don't have the memory - mem_percent = mem_status.percent - if mem_percent > max_mem_percent: - free_memory = mem_status.available - free_memory_human = self.helpers.bytes_to_human(free_memory) - self.hugewarning( - f"Cannot proceed with DNS mutations because system memory is at {mem_percent:.1f}% ({free_memory_human} remaining)" - ) - break + try: + for i, (domain, subdomains) in enumerate(trimmed_found): + self.verbose(f"{domain} has {len(subdomains):,} subdomains") + # keep looping as long as we're finding things + while 1: + max_mem_percent = 90 + mem_status = self.helpers.memory_status() + # abort if we don't have the memory + mem_percent = mem_status.percent + if mem_percent > max_mem_percent: + free_memory = mem_status.available + free_memory_human = self.helpers.bytes_to_human(free_memory) + assert ( + False + ), f"Cannot proceed with DNS mutations because system memory is at {mem_percent:.1f}% ({free_memory_human} remaining)" - query = domain - domain_hash = hash(domain) - if self.scan.stopping: - return + query = domain + domain_hash = hash(domain) + if self.scan.stopping: + return - mutations = set(base_mutations) + mutations = set(base_mutations) - def add_mutation(_domain_hash, m): - h = hash((_domain_hash, m)) - if h not in self.mutations_tried: - self.mutations_tried.add(h) - mutations.add(m) + def add_mutation(_domain_hash, m): + h = hash((_domain_hash, m)) + if h not in self.mutations_tried: + self.mutations_tried.add(h) + mutations.add(m) - # try every subdomain everywhere else - for _domain, _subdomains in found: - if _domain == domain: - continue - for s in _subdomains: - first_segment = s.split(".")[0] - # skip stuff with lots of numbers (e.g. PTRs) - digits = self.digit_regex.findall(first_segment) - excessive_digits = len(digits) > 2 - long_digits = any(len(d) > 3 for d in digits) - if excessive_digits or long_digits: + # try every subdomain everywhere else + for _domain, _subdomains in found: + if _domain == domain: continue - add_mutation(domain_hash, first_segment) - for word in self.helpers.extract_words( - first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes - ): - add_mutation(domain_hash, word) + for s in _subdomains: + first_segment = s.split(".")[0] + # skip stuff with lots of numbers (e.g. PTRs) + digits = self.digit_regex.findall(first_segment) + excessive_digits = len(digits) > 2 + long_digits = any(len(d) > 3 for d in digits) + if excessive_digits or long_digits: + continue + add_mutation(domain_hash, first_segment) + for word in self.helpers.extract_words( + first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes + ): + add_mutation(domain_hash, word) - # word cloud - for mutation in self.helpers.word_cloud.mutations( - subdomains, cloud=False, numbers=3, number_padding=1 - ): - for delimiter in ("", ".", "-"): - m = delimiter.join(mutation).lower() - add_mutation(domain_hash, m) + # numbers + devops mutations + for mutation in self.helpers.word_cloud.mutations( + subdomains, cloud=False, numbers=3, number_padding=1 + ): + for delimiter in ("", ".", "-"): + m = delimiter.join(mutation).lower() + add_mutation(domain_hash, m) - # special dns mutator - for subdomain in self.helpers.word_cloud.dns_mutator.mutations( - subdomains, max_mutations=self.max_mutations - ): - add_mutation(domain_hash, subdomain) + # special dns mutator + for subdomain in self.helpers.word_cloud.dns_mutator.mutations( + subdomains, max_mutations=self.max_mutations + ): + add_mutation(domain_hash, subdomain) - if mutations: - self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})") - results = list(self.massdns(query, mutations)) - for hostname in results: - source_event = self.get_source_event(hostname) - if source_event is None: - self.warning(f"Could not correlate source event from: {hostname}") + if mutations: + self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})") + results = list(self.massdns(query, mutations)) + for hostname in results: + source_event = self.get_source_event(hostname) + if source_event is None: + self.warning(f"Could not correlate source event from: {hostname}") + continue + self.emit_result(hostname, source_event, query) + if results: continue - self.emit_result(hostname, source_event, query) - if results: - continue - break + break + except AssertionError as e: + self.warning(e) def add_found(self, host): if not isinstance(host, str): diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index aa18483ae9..81ecc8ca8c 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -52,7 +52,7 @@ def handle_event(self, event): "IP Address": ",".join( str(x) for x in getattr(event, "resolved_hosts", set()) if self.helpers.is_ip(x) ), - "Source Module": str(getattr(event, "module", "")), + "Source Module": str(getattr(event, "module_sequence", "")), "Scope Distance": str(getattr(event, "scope_distance", "")), "Event Tags": ",".join(sorted(list(getattr(event, "tags", [])))), } diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py index b9eef9402e..61f403c095 100644 --- a/bbot/modules/output/human.py +++ b/bbot/modules/output/human.py @@ -20,7 +20,7 @@ def handle_event(self, event): event_tags = "" if getattr(event, "tags", []): event_tags = f'\t({", ".join(sorted(getattr(event, "tags", [])))})' - event_str = f"{event_type:<20}\t{event.data_human}\t{event.module}{event_tags}" + event_str = f"{event_type:<20}\t{event.data_human}\t{event.module_sequence}{event_tags}" # log vulnerabilities in vivid colors if event.type == "VULNERABILITY": severity = event.data.get("severity", "INFO") From 839ef0a8de33b8a41da0b979fd44baa5c3d787b3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 May 2023 16:59:59 -0400 Subject: [PATCH 002/387] division by zero --- bbot/modules/massdns.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 10b69aa0cd..718dc0eb58 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -234,16 +234,17 @@ def _massdns(self, domain, subdomains): def finish(self): found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True) # if we have a lot of rounds to make, don't try mutations on less-populated domains - avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50]) trimmed_found = [] - for i, (domain, subdomains) in enumerate(found): - # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains - if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)): - trimmed_found.append((domain, subdomains)) - else: - self.verbose( - f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})" - ) + if found: + avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50]) + for i, (domain, subdomains) in enumerate(found): + # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains + if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)): + trimmed_found.append((domain, subdomains)) + else: + self.verbose( + f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})" + ) base_mutations = set() try: From 1400e6c94cc6098a7fbe5c227cba6b8dde60f712 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 5 May 2023 19:26:08 -0400 Subject: [PATCH 003/387] set status frequency default to 15 and made configurable --- bbot/defaults.yml | 2 ++ bbot/scanner/manager.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index a22c296cc8..6de4c44c48 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -75,6 +75,8 @@ dns_filter_ptrs: true dns_debug: false # Whether to verify SSL certificates ssl_verify: false +# Interval for displaying status messages +status_frequency: 15 # How many scan results to keep before cleaning up the older ones keep_scans: 20 # Completely ignore URLs with these extensions diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 913c24f5e8..fd7d585daf 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -34,6 +34,8 @@ def __init__(self, scan): self.events_resolved = dict() self.dns_resolution = self.scan.config.get("dns_resolution", False) + self.status_frequency = self.scan.config.get("status_frequency", 15) + self.last_log_time = datetime.now() def init_events(self): @@ -407,7 +409,7 @@ def loop_until_finished(self): yield from events try: - self.log_status() + self.log_status(self.status_frequency) event, kwargs = self.incoming_event_queue.get_nowait() while not self.scan.aborting: try: @@ -416,7 +418,7 @@ def loop_until_finished(self): activity = True break except queue.Full: - self.log_status() + self.log_status(self.status_frequency) with self.event_emitted: self.event_emitted.wait(timeout=0.1) except queue.Empty: @@ -451,8 +453,8 @@ def loop_until_finished(self): for mod in self.scan.modules.values(): self.catch(mod._register_running, mod.report, _force=True) - def log_status(self, frequency=10): - # print status every 10 seconds + def log_status(self, frequency=15): + # print status every 15 seconds (or status_frequency setting) timedelta_secs = timedelta(seconds=frequency) now = datetime.now() time_since_last_log = now - self.last_log_time From 91b4f988358d04da591f491edd22409e75db4bbd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 5 May 2023 09:44:37 -0400 Subject: [PATCH 004/387] basic scan working (no modules) --- bbot/cli.py | 33 ++- bbot/core/event/base.py | 6 +- bbot/core/helpers/async_helpers.py | 46 ++++ bbot/core/helpers/dns.py | 239 +++++++----------- bbot/core/helpers/helper.py | 5 - bbot/core/helpers/misc.py | 4 + bbot/core/helpers/threadpool.py | 266 -------------------- bbot/modules/base.py | 166 +++++-------- bbot/modules/internal/aggregate.py | 2 +- bbot/modules/internal/excavate.py | 4 +- bbot/modules/internal/speculate.py | 4 +- bbot/modules/output/asset_inventory.py | 4 +- bbot/modules/output/csv.py | 11 +- bbot/modules/output/http.py | 2 +- bbot/modules/output/human.py | 11 +- bbot/modules/output/json.py | 11 +- bbot/modules/output/neo4j.py | 2 +- bbot/modules/output/web_report.py | 2 +- bbot/modules/output/websocket.py | 2 +- bbot/modules/report/asn.py | 2 +- bbot/scanner/manager.py | 324 ++++++++++--------------- bbot/scanner/scanner.py | 192 +++++++-------- bbot/test/test.conf | 2 +- bbot/test/test_step_2/test_helpers.py | 104 +------- bbot/test/test_step_2/test_manager.py | 20 ++ poetry.lock | 21 +- pyproject.toml | 1 + 27 files changed, 506 insertions(+), 980 deletions(-) create mode 100644 bbot/core/helpers/async_helpers.py delete mode 100644 bbot/core/helpers/threadpool.py diff --git a/bbot/cli.py b/bbot/cli.py index ea8f305690..55ee624f3d 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -2,6 +2,7 @@ import os import sys +import asyncio import logging import threading import traceback @@ -31,9 +32,12 @@ from . import config -def main(): +scan_name = "" + + +async def _main(): err = False - scan_name = "" + global scan_name ensure_config_files() @@ -258,7 +262,7 @@ def main(): scanner.helpers.word_cloud.load(options.load_wordcloud) - scanner.prep() + await scanner.prep() if not options.dry_run: if not options.agent_mode and not options.yes and sys.stdin.isatty(): @@ -282,7 +286,7 @@ def keyboard_listen(): keyboard_listen_thread = threading.Thread(target=keyboard_listen, daemon=True) keyboard_listen_thread.start() - scanner.start_without_generator() + await scanner.start_without_generator() except bbot.core.errors.ScanError as e: log_to_stderr(str(e), level="ERROR") @@ -290,7 +294,7 @@ def keyboard_listen(): raise finally: with suppress(NameError): - scanner.cleanup() + await scanner.cleanup() except bbot.core.errors.BBOTError as e: log_to_stderr(f"{e} (--debug for details)", level="ERROR") @@ -302,13 +306,6 @@ def keyboard_listen(): log_to_stderr(f"Encountered unknown error: {traceback.format_exc()}", level="ERROR") err = True - except KeyboardInterrupt: - msg = "Interrupted" - if scan_name: - msg = f"You killed {scan_name}" - log_to_stderr(msg, level="ERROR") - err = True - finally: # save word cloud with suppress(BaseException): @@ -330,5 +327,17 @@ def keyboard_listen(): """ +def main(): + global scan_name + try: + asyncio.run(_main()) + except KeyboardInterrupt: + msg = "Interrupted" + if scan_name: + msg = f"You killed {scan_name}" + log_to_stderr(msg, level="ERROR") + err = True + + if __name__ == "__main__": main() diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index a88c5314fc..f96e4a79a2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,4 +1,5 @@ import json +import asyncio import logging import ipaddress import traceback @@ -6,7 +7,6 @@ from datetime import datetime from contextlib import suppress from pydantic import BaseModel, validator -from threading import Event as ThreadingEvent from .helpers import * from bbot.core.errors import * @@ -125,8 +125,8 @@ def __init__( if _internal: # or source._internal: self.make_internal() - # a threading event indicating whether the event has undergone DNS resolution yet - self._resolved = ThreadingEvent() + # an event indicating whether the event has undergone DNS resolution + self._resolved = asyncio.Event() @property def data(self): diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py new file mode 100644 index 0000000000..9a7b7bcde1 --- /dev/null +++ b/bbot/core/helpers/async_helpers.py @@ -0,0 +1,46 @@ +import asyncio +import logging +from contextlib import asynccontextmanager + +log = logging.getLogger("bbot.core.helpers.async_helpers") + +from .cache import CacheDict + + +class _Lock(asyncio.Lock): + def __init__(self, name): + self.name = name + super().__init__() + + +class NamedLock: + """ + Returns a unique asyncio.Lock() based on a provided string + + Useful for preventing multiple operations from occuring on the same data in parallel + E.g. simultaneous DNS lookups on the same hostname + """ + + def __init__(self, max_size=1000): + self._cache = CacheDict(max_size=max_size) + + @asynccontextmanager + async def lock(self, name): + try: + lock = self._cache.get(name) + except KeyError: + lock = _Lock(name) + self._cache.put(name, lock) + async with lock: + yield + + +class TaskCounter: + def __init__(self): + self.value = 0 + + def __enter__(self): + self.value += 1 + + def __exit__(self, exc_type, exc_val, exc_tb): + self.value -= 1 diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index cd3686e23e..1089206a61 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -1,15 +1,15 @@ import json +import asyncio import logging import ipaddress import traceback -import dns.resolver import dns.exception -from threading import Lock +import dns.asyncresolver from contextlib import suppress from .regexes import dns_name_regex +from bbot.core.helpers.async_helpers import NamedLock from bbot.core.errors import ValidationError, DNSError -from .threadpool import NamedLock, PatchedThreadPoolExecutor from .misc import is_ip, is_domain, domain_parents, parent_domain, rand_string, cloudcheck log = logging.getLogger("bbot.core.helpers.dns") @@ -20,13 +20,12 @@ class DNSHelper: For automatic wildcard detection, nameserver validation, etc. """ - nameservers_url = "https://public-dns.info/nameserver/nameservers.json" all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] def __init__(self, parent_helper): self.parent_helper = parent_helper try: - self.resolver = dns.resolver.Resolver() + self.resolver = dns.asyncresolver.Resolver() except Exception as e: raise DNSError(f"Failed to create BBOT DNS resolver: {e}") self.timeout = self.parent_helper.config.get("dns_timeout", 5) @@ -48,25 +47,13 @@ def __init__(self, parent_helper): self._wildcard_lock = NamedLock() # keeps track of warnings issued for wildcard detection to prevent duplicate warnings self._dns_warnings = set() - self._errors = dict() - self._error_lock = Lock() - self.fallback_nameservers_file = self.parent_helper.wordlist_dir / "nameservers.txt" - - # we need our own threadpool because using the shared one can lead to deadlocks - max_workers = self.parent_helper.config.get("max_dns_threads", 100) - self._thread_pool = PatchedThreadPoolExecutor(max_workers=max_workers) - + self.max_threads = self.parent_helper.config.get("max_dns_threads", 100) self._debug = self.parent_helper.config.get("dns_debug", False) - self._dummy_modules = dict() - self._dummy_modules_lock = Lock() - self._dns_cache = self.parent_helper.CacheDict(max_size=100000) - self._event_cache = self.parent_helper.CacheDict(max_size=10000) - self._event_cache_lock = Lock() self._event_cache_locks = NamedLock() # copy the system's current resolvers to a text file for tool use @@ -75,7 +62,7 @@ def __init__(self, parent_helper): self.filter_bad_ptrs = self.parent_helper.config.get("dns_filter_ptrs", True) - def resolve(self, query, **kwargs): + async def resolve(self, query, **kwargs): """ "1.2.3.4" --> { "evilcorp.com", @@ -86,14 +73,14 @@ def resolve(self, query, **kwargs): } """ results = set() - raw_results, errors = self.resolve_raw(query, **kwargs) + raw_results, errors = await self.resolve_raw(query, **kwargs) for rdtype, answers in raw_results: for answer in answers: for _, t in self.extract_targets(answer): results.add(t) return results - def resolve_raw(self, query, **kwargs): + async def resolve_raw(self, query, **kwargs): # DNS over TCP is more reliable # But setting this breaks DNS resolution on Ubuntu because systemd-resolve doesn't support TCP # kwargs["tcp"] = True @@ -101,7 +88,7 @@ def resolve_raw(self, query, **kwargs): if is_ip(query): kwargs.pop("type", None) kwargs.pop("rdtype", None) - results, errors = self._resolve_ip(query, **kwargs) + results, errors = await self._resolve_ip(query, **kwargs) return [("PTR", results)], [("PTR", e) for e in errors] else: results = [] @@ -118,7 +105,7 @@ def resolve_raw(self, query, **kwargs): elif any([isinstance(t, x) for x in (list, tuple)]): types = [str(_).strip().upper() for _ in t] for t in types: - r, e = self._resolve_hostname(query, rdtype=t, **kwargs) + r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) if r: results.append((t, r)) for error in e: @@ -126,13 +113,7 @@ def resolve_raw(self, query, **kwargs): return (results, errors) - def submit_task(self, *args, **kwargs): - try: - return self._thread_pool.submit(*args, **kwargs) - except RuntimeError as e: - log.debug(f"Error submitting DNS thread task: {e}") - - def _resolve_hostname(self, query, **kwargs): + async def _resolve_hostname(self, query, **kwargs): self.debug(f"Resolving {query} with kwargs={kwargs}") results = [] errors = [] @@ -144,8 +125,8 @@ def _resolve_hostname(self, query, **kwargs): parent_hash = hash(f"{parent}:{rdtype}") dns_cache_hash = hash(f"{query}:{rdtype}") while tries_left > 0: - if self.parent_helper.scan_stopping: - break + # if self.parent_helper.scan_stopping: + # break try: try: results = self._dns_cache[dns_cache_hash] @@ -156,19 +137,17 @@ def _resolve_hostname(self, query, **kwargs): f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' ) return results, errors - results = list(self._catch(self.resolver.resolve, query, **kwargs)) + results = await self._catch(self.resolver.resolve, query, **kwargs) if cache_result: self._dns_cache[dns_cache_hash] = results - with self._error_lock: - if parent_hash in self._errors: - self._errors[parent_hash] = 0 + if parent_hash in self._errors: + self._errors[parent_hash] = 0 break except (dns.resolver.NoNameservers, dns.exception.Timeout, dns.resolver.LifetimeTimeout) as e: - with self._error_lock: - try: - self._errors[parent_hash] += 1 - except KeyError: - self._errors[parent_hash] = 1 + try: + self._errors[parent_hash] += 1 + except KeyError: + self._errors[parent_hash] = 1 errors.append(e) # don't retry if we get a SERVFAIL if isinstance(e, dns.resolver.NoNameservers): @@ -184,10 +163,9 @@ def _resolve_hostname(self, query, **kwargs): else: log.verbose(err_msg) - self.debug(f"Results for {query} with kwargs={kwargs}: {results}") return results, errors - def _resolve_ip(self, query, **kwargs): + async def _resolve_ip(self, query, **kwargs): self.debug(f"Reverse-resolving {query} with kwargs={kwargs}") retries = kwargs.pop("retries", 0) cache_result = kwargs.pop("cache_result", False) @@ -196,13 +174,13 @@ def _resolve_ip(self, query, **kwargs): errors = [] dns_cache_hash = hash(f"{query}:PTR") while tries_left > 0: - if self.parent_helper.scan_stopping: - break + # if self.parent_helper.scan_stopping: + # break try: if dns_cache_hash in self._dns_cache: result = self._dns_cache[dns_cache_hash] else: - result = list(self._catch(self.resolver.resolve_address, query, **kwargs)) + result = await self._catch(self.resolver.resolve_address, query, **kwargs) if cache_result: self._dns_cache[dns_cache_hash] = result return result, errors @@ -220,21 +198,21 @@ def _resolve_ip(self, query, **kwargs): self.debug(f"Results for {query} with kwargs={kwargs}: {results}") return results, errors - def handle_wildcard_event(self, event, children): + async def handle_wildcard_event(self, event, children): event_host = str(event.host) # wildcard checks if not is_ip(event.host): # check if this domain is using wildcard dns event_target = "target" in event.tags - for hostname, wildcard_domain_rdtypes in self.is_wildcard_domain( - event_host, log_info=event_target + for hostname, wildcard_domain_rdtypes in ( + await self.is_wildcard_domain(event_host, log_info=event_target) ).items(): if wildcard_domain_rdtypes: event.add_tag("wildcard-domain") for rdtype, ips in wildcard_domain_rdtypes.items(): event.add_tag(f"{rdtype.lower()}-wildcard-domain") # check if the dns name itself is a wildcard entry - wildcard_rdtypes = self.is_wildcard(event_host) + wildcard_rdtypes = await self.is_wildcard(event_host) for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): wildcard_tag = "error" if is_wildcard == True: @@ -263,7 +241,7 @@ def handle_wildcard_event(self, event, children): log.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') event.data = wildcard_data - def resolve_event(self, event, minimal=False): + async def resolve_event(self, event, minimal=False): """ Tag event with appropriate dns record types Optionally create child events from dns resolutions @@ -279,7 +257,7 @@ def resolve_event(self, event, minimal=False): return event_tags, event_whitelisted, event_blacklisted, dns_children # lock to ensure resolution of the same host doesn't start while we're working here - with self._event_cache_locks.get_lock(event_host): + async with self._event_cache_locks.lock(event_host): # try to get data from cache _event_tags, _event_whitelisted, _event_blacklisted, _dns_children = self.event_cache_get(event_host) event_tags.update(_event_tags) @@ -299,17 +277,9 @@ def resolve_event(self, event, minimal=False): types = ("A", "AAAA") if types: - futures = {} - for t in types: - future = self.submit_task( - self._catch_keyboardinterrupt, self.resolve_raw, event_host, type=t, cache_result=True - ) - if future is None: - break - futures[future] = t - - for future in self.parent_helper.as_completed(futures): - resolved_raw, errors = future.result() + tasks = [self.resolve_raw(event_host, type=t, cache_result=True) for t in types] + for task in asyncio.as_completed(tasks): + resolved_raw, errors = await task for rdtype, e in errors: if rdtype not in resolved_raw: event_tags.add(f"{rdtype.lower()}-error") @@ -380,22 +350,23 @@ def event_cache_get(self, host): except KeyError: return set(), None, None, set() - def resolve_batch(self, queries, **kwargs): + async def resolve_batch(self, queries, **kwargs): """ - resolve_batch("www.evilcorp.com", "evilcorp.com") --> [ + await resolve_batch("www.evilcorp.com", "evilcorp.com") --> [ ("www.evilcorp.com", {"1.1.1.1"}), ("evilcorp.com", {"2.2.2.2"}) ] """ - futures = dict() - for query in queries: - future = self.submit_task(self._catch_keyboardinterrupt, self.resolve, query, **kwargs) - if future is None: - break - futures[future] = query - for future in self.parent_helper.as_completed(futures): - query = futures[future] - yield (query, future.result()) + + async def coro_wrapper(q, **_kwargs): + """ + Helps us correlate task results back to their original arguments + """ + result = await self.resolve(q, **_kwargs) + return (q, result) + + for task in asyncio.as_completed([coro_wrapper(q, **kwargs) for q in queries]): + yield await task def extract_targets(self, record): """ @@ -424,44 +395,13 @@ def extract_targets(self, record): @staticmethod def _clean_dns_record(record): - with suppress(Exception): + if not isinstance(record, str): record = str(record.to_text()) return str(record).rstrip(".").lower() - def get_valid_resolvers(self, min_reliability=0.99): - nameservers = set() - nameservers_file = self.parent_helper.download(self.nameservers_url, cache_hrs=72) - if nameservers_file is None: - log.warning(f"Failed to download nameservers from {self.nameservers_url}") - else: - nameservers_json = [] - try: - nameservers_json = json.loads(open(nameservers_file).read()) - except Exception as e: - log.warning(f"Failed to load nameserver list from {nameservers_file}: {e}") - nameservers_file.unlink() - for entry in nameservers_json: - try: - ip = str(entry.get("ip", "")).strip() - except Exception: - continue - try: - reliability = float(entry.get("reliability", 0)) - except ValueError: - continue - if reliability >= min_reliability and is_ip(ip, version=4): - nameservers.add(ip) - log.verbose(f"Loaded {len(nameservers):,} nameservers from {self.nameservers_url}") - if not nameservers: - log.info(f"Loading fallback nameservers from {self.fallback_nameservers_file}") - lines = self.parent_helper.read_file(self.fallback_nameservers_file) - nameservers = set([l for l in lines if not l.startswith("#")]) - resolver_list = self.verify_nameservers(nameservers) - return resolver_list - - def _catch(self, callback, *args, **kwargs): + async def _catch(self, callback, *args, **kwargs): try: - return callback(*args, **kwargs) + return await callback(*args, **kwargs) except dns.resolver.NoNameservers: raise except (dns.exception.Timeout, dns.resolver.LifetimeTimeout): @@ -473,7 +413,7 @@ def _catch(self, callback, *args, **kwargs): log.warning(f"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}") return list() - def is_wildcard(self, query, ips=None, rdtype=None): + async def is_wildcard(self, query, ips=None, rdtype=None): """ Use this method to check whether a *host* is a wildcard entry @@ -507,31 +447,27 @@ def is_wildcard(self, query, ips=None, rdtype=None): parent = parent_domain(query) parents = list(domain_parents(query)) - futures = [] + wildcard_tasks = {t: [] for t in self.all_rdtypes} base_query_ips = dict() # if the caller hasn't already done the work of resolving the IPs if ips is None: # then resolve the query for all rdtypes for _rdtype in self.all_rdtypes: # resolve the base query - future = self.submit_task( - self._catch_keyboardinterrupt, self.resolve_raw, query, type=_rdtype, cache_result=True - ) - if future is None: - break - futures.append(future) - - for future in self.parent_helper.as_completed(futures): - raw_results, errors = future.result() - if errors and not raw_results: - self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection") - result[_rdtype] = (None, parent) - continue - for _rdtype, answers in raw_results: - base_query_ips[_rdtype] = set() - for answer in answers: - for _, t in self.extract_targets(answer): - base_query_ips[_rdtype].add(t) + wildcard_tasks[_rdtype].append(self.resolve_raw(query, type=_rdtype, cache_result=True)) + + for _rdtype, tasks in wildcard_tasks.items(): + for task in asyncio.as_completed(tasks): + raw_results, errors = await task + if errors and not raw_results: + self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection") + result[_rdtype] = (None, parent) + continue + for __rdtype, answers in raw_results: + base_query_ips[__rdtype] = set() + for answer in answers: + for _, t in self.extract_targets(answer): + base_query_ips[__rdtype].add(t) else: # otherwise, we can skip all that base_query_ips[rdtype] = set([self._clean_dns_record(ip) for ip in ips]) @@ -550,7 +486,7 @@ def is_wildcard(self, query, ips=None, rdtype=None): for host in parents[::-1]: host_hash = hash(host) # make sure we've checked that domain for wildcards - self.is_wildcard_domain(host) + await self.is_wildcard_domain(host) if host_hash in self._wildcard_cache: # then get its IPs from our wildcard cache wildcard_rdtypes = self._wildcard_cache[host_hash] @@ -565,7 +501,7 @@ def is_wildcard(self, query, ips=None, rdtype=None): return result - def is_wildcard_domain(self, domain, log_info=False): + async def is_wildcard_domain(self, domain, log_info=False): """ Check whether a domain is using wildcard DNS @@ -581,14 +517,14 @@ def is_wildcard_domain(self, domain, log_info=False): for i, host in enumerate(parents[::-1]): # have we checked this host before? host_hash = hash(host) - with self._wildcard_lock.get_lock(host_hash): + async with self._wildcard_lock.lock(host_hash): # if we've seen this host before if host_hash in self._wildcard_cache: wildcard_domain_results[host] = self._wildcard_cache[host_hash] continue # determine if this is a wildcard domain - wildcard_futures = {} + wildcard_tasks = {t: [] for t in self.all_rdtypes} # resolve a bunch of random subdomains of the same parent for rdtype in self.all_rdtypes: # continue if a wildcard was already found for this rdtype @@ -596,29 +532,19 @@ def is_wildcard_domain(self, domain, log_info=False): # continue for _ in range(self.wildcard_tests): rand_query = f"{rand_string(digits=False, length=10)}.{host}" - future = self.submit_task( - self._catch_keyboardinterrupt, - self.resolve, - rand_query, - type=rdtype, - cache_result=False, - ) - if future is None: - break - wildcard_futures[future] = rdtype + wildcard_tasks[rdtype].append(self.resolve(rand_query, type=rdtype, cache_result=False)) # combine the random results is_wildcard = False wildcard_results = dict() - for future in self.parent_helper.as_completed(wildcard_futures): - results = future.result() - rdtype = wildcard_futures[future] - if results: - is_wildcard = True - if results: - if not rdtype in wildcard_results: - wildcard_results[rdtype] = set() - wildcard_results[rdtype].update(results) + for rdtype, tasks in wildcard_tasks.items(): + for task in asyncio.as_completed(tasks): + results = await task + if results: + is_wildcard = True + if not rdtype in wildcard_results: + wildcard_results[rdtype] = set() + wildcard_results[rdtype].update(results) self._wildcard_cache.update({host_hash: wildcard_results}) wildcard_domain_results.update({host: wildcard_results}) @@ -646,12 +572,11 @@ def debug(self, *args, **kwargs): log.debug(*args, **kwargs) def _get_dummy_module(self, name): - with self._dummy_modules_lock: - try: - dummy_module = self._dummy_modules[name] - except KeyError: - dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") - self._dummy_modules[name] = dummy_module + try: + dummy_module = self._dummy_modules[name] + except KeyError: + dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") + self._dummy_modules[name] = dummy_module return dummy_module def dns_warning(self, msg): diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index fda18fa745..db6f1b74a8 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -9,7 +9,6 @@ from .wordcloud import WordCloud from .cloud import CloudProviders from .interactsh import Interactsh -from .threadpool import as_completed from ...scanner.target import Target from ...modules.base import BaseModule from .depsinstaller import DepsInstaller @@ -88,10 +87,6 @@ def scan_stopping(self): def in_tests(self): return os.environ.get("BBOT_TESTING", "") == "True" - @staticmethod - def as_completed(*args, **kwargs): - return as_completed(*args, **kwargs) - def _make_dummy_module(self, name, _type="scan"): """ Construct a dummy module, for attachment to events diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index e3637d2187..e65766843a 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1020,3 +1020,7 @@ def cloudcheck(ip): with suppress(KeyError): provider = provider_map[provider.lower()] return provider, provider_type, subnet + + +def is_async_function(f): + return inspect.iscoroutinefunction(f) diff --git a/bbot/core/helpers/threadpool.py b/bbot/core/helpers/threadpool.py deleted file mode 100644 index ef5ca8d4ec..0000000000 --- a/bbot/core/helpers/threadpool.py +++ /dev/null @@ -1,266 +0,0 @@ -import queue -import logging -import threading -import traceback -from datetime import datetime -from queue import SimpleQueue, Full -from concurrent.futures import ThreadPoolExecutor - -log = logging.getLogger("bbot.core.helpers.threadpool") - -from .cache import CacheDict -from ...core.errors import ScanCancelledError - - -def pretty_fn(a): - if callable(a): - return a.__qualname__ - return a - - -class ThreadPoolSimpleQueue(SimpleQueue): - def __init__(self, *args, **kwargs): - self._executor = kwargs.pop("_executor", None) - assert self._executor is not None, "Must specify _executor" - - def get(self, *args, **kwargs): - work_item = super().get(*args, **kwargs) - if work_item is not None: - thread_id = threading.get_ident() - self._executor._current_work_items[thread_id] = (work_item, datetime.now()) - return work_item - - -class PatchedThreadPoolExecutor(ThreadPoolExecutor): - """ - This class exists only because of a bug in cpython where - futures are not properly marked CANCELLED_AND_NOTIFIED: - https://github.com/python/cpython/issues/87893 - """ - - def shutdown(self, wait=True, *, cancel_futures=False): - with self._shutdown_lock: - self._shutdown = True - if cancel_futures: - # Drain all work items from the queue, and then cancel their - # associated futures. - while 1: - try: - work_item = self._work_queue.get_nowait() - except queue.Empty: - break - if work_item is not None: - if work_item.future.cancel(): - work_item.future.set_running_or_notify_cancel() - - # Send a wake-up to prevent threads calling - # _work_queue.get(block=True) from permanently blocking. - self._work_queue.put(None) - if wait: - for t in self._threads: - t.join() - - -class BBOTThreadPoolExecutor(PatchedThreadPoolExecutor): - """ - Allows inspection of thread pool to determine which functions are currently executing - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._current_work_items = {} - self._work_queue = ThreadPoolSimpleQueue(_executor=self) - - @property - def threads_status(self): - work_items = [] - for thread_id, (work_item, start_time) in sorted(self._current_work_items.items()): - func = work_item.fn.__qualname__ - func_index = 0 - if work_item and not work_item.future.done(): - for i, f in enumerate(list(work_item.args)): - if callable(f): - func = f.__qualname__ - func_index = i + 1 - else: - break - running_for = datetime.now() - start_time - wi_args = list(work_item.args)[func_index:] - wi_args = [pretty_fn(a) for a in wi_args] - wi_args = str(wi_args).strip("[]") - wi_kwargs = ", ".join(["{0}={1}".format(k, pretty_fn(v)) for k, v in work_item.kwargs.items()]) - func_with_args = f"{func}({wi_args}" + (f", {wi_kwargs}" if wi_kwargs else "") + ")" - work_items.append( - (running_for, f"running for {int(running_for.total_seconds()):>3} seconds: {func_with_args}") - ) - work_items.sort(key=lambda x: x[0]) - return [x[-1] for x in work_items] - - -class ThreadPoolWrapper: - """ - Layers more granular control overtop of a shared thread pool - Allows setting lower thread limits for modules, etc. - """ - - def __init__(self, executor, max_workers=None, qsize=None): - self.executor = executor - self.max_workers = max_workers - self.max_qsize = qsize - self.futures = set() - - self._num_tasks = 0 - self._task_count_lock = threading.Lock() - - self._lock = threading.RLock() - self.not_full = threading.Condition(self._lock) - - try: - self.executor._thread_pool_wrappers.append(self) - except AttributeError: - self.executor._thread_pool_wrappers = [self] - - def submit_task(self, callback, *args, **kwargs): - """ - A wrapper around threadpool.submit() - """ - block = kwargs.pop("_block", True) - force = kwargs.pop("_force_submit", False) - success = False - with self.not_full: - self.num_tasks_increment() - try: - if not force: - if not block: - if self.is_full or self.underlying_executor_is_full: - raise Full - else: - # wait until there's room - while self.is_full or self.underlying_executor_is_full: - self.not_full.wait() - - try: - # submit the job - future = self.executor.submit(self._execute_callback, callback, *args, **kwargs) - future.add_done_callback(self._on_future_done) - success = True - return future - except RuntimeError as e: - raise ScanCancelledError(e) - finally: - if not success: - self.num_tasks_decrement() - - def _execute_callback(self, callback, *args, **kwargs): - try: - return callback(*args, **kwargs) - finally: - self.num_tasks_decrement() - - def _on_future_done(self, future): - if future.cancelled(): - self.num_tasks_decrement() - - @property - def num_tasks(self): - with self._task_count_lock: - return self._num_tasks - - def num_tasks_increment(self): - with self._task_count_lock: - self._num_tasks += 1 - - def num_tasks_decrement(self): - with self._task_count_lock: - self._num_tasks = max(0, self._num_tasks - 1) - for wrapper in self.executor._thread_pool_wrappers: - try: - with wrapper.not_full: - wrapper.not_full.notify() - except RuntimeError: - continue - except Exception as e: - log.warning(f"Unknown error in num_tasks_decrement(): {e}") - log.trace(traceback.format_exc()) - - @property - def is_full(self): - if self.max_workers is None: - return False - return self.num_tasks > self.max_workers - - @property - def underlying_executor_is_full(self): - return self.max_qsize is not None and self.qsize >= self.max_qsize - - @property - def qsize(self): - return self.executor._work_queue.qsize() - - def shutdown(self, *args, **kwargs): - self.executor.shutdown(*args, **kwargs) - - @property - def threads_status(self): - return self.executor.threads_status - - -from concurrent.futures._base import FINISHED -from concurrent.futures._base import as_completed as as_completed_orig - - -def as_completed(*args, **kwargs): - for f in as_completed_orig(*args, **kwargs): - if f._state == FINISHED: - yield f - - -class _Lock: - def __init__(self, name): - self.name = name - self.lock = threading.Lock() - - def __enter__(self): - self.lock.acquire() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.lock.release() - - -class NamedLock: - """ - Returns a unique threading.Lock() based on a provided string - - Useful for preventing multiple operations from occuring on the same data in parallel - E.g. simultaneous DNS lookups on the same hostname - """ - - def __init__(self, max_size=1000): - self._cache = CacheDict(max_size=max_size) - - def get_lock(self, name): - try: - return self._cache.get(name) - except KeyError: - new_lock = _Lock(name) - self._cache.put(name, new_lock) - return new_lock - - -class TaskCounter: - def __init__(self): - self._value = 0 - self.lock = threading.Lock() - - def __enter__(self): - with self.lock: - self._value += 1 - - def __exit__(self, exc_type, exc_val, exc_tb): - with self.lock: - self._value -= 1 - - @property - def value(self): - with self.lock: - return self._value diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 8e1728d6c6..888a56f9a7 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1,12 +1,12 @@ import queue +import asyncio import logging import threading -import traceback from sys import exc_info from contextlib import suppress from ..core.helpers.misc import get_size -from ..core.helpers.threadpool import ThreadPoolWrapper, TaskCounter +from ..core.helpers.async_helpers import TaskCounter from ..core.errors import ScanCancelledError, ValidationError, WordlistError @@ -76,28 +76,21 @@ class BaseModule: # Type, for differentiating between normal modules and output modules, etc. _type = "scan" - _report_lock = threading.Lock() - def __init__(self, scan): self.scan = scan self.errored = False self._log = None self._incoming_event_queue = None # seconds since we've submitted a batch + self._outgoing_event_queue = None + # seconds since we've submitted a batch self._last_submitted_batch = None - # wrapper around shared thread pool to ensure that a single module doesn't hog more than its share - self.thread_pool = ThreadPoolWrapper(self.scan._thread_pool) - self._internal_thread_pool = ThreadPoolWrapper( - self.scan._internal_thread_pool.executor, max_workers=self.max_event_handlers - ) # additional callbacks to be executed alongside self.cleanup() self.cleanup_callbacks = [] self._cleanedup = False self._watched_events = None - self._lock = threading.RLock() - self._running_counter = TaskCounter() - self.event_received = threading.Condition(self._lock) + self._task_counter = TaskCounter() # string constant self._custom_filter_criteria_msg = "it did not meet custom filter criteria" @@ -105,7 +98,7 @@ def __init__(self, scan): # track number of failures (for .request_with_fail_count()) self._request_failures = 0 - def setup(self): + async def setup(self): """ Perform setup functions at the beginning of the scan. Optionally override this method. @@ -114,7 +107,7 @@ def setup(self): """ return True - def handle_event(self, event): + async def handle_event(self, event): """ Override this method if batch_size == 1. """ @@ -135,7 +128,7 @@ def filter_event(self, event): """ return True - def finish(self): + async def finish(self): """ Perform final functions when scan is nearing completion @@ -147,7 +140,7 @@ def finish(self): """ return - def report(self): + async def report(self): """ Perform a final task when the scan is finished, but before cleanup happens @@ -155,7 +148,7 @@ def report(self): """ return - def cleanup(self): + async def cleanup(self): """ Perform final cleanup after the scan has finished This method is called only once, and may not raise events. @@ -211,21 +204,21 @@ def submit_task(self, *args, **kwargs): kwargs["_block"] = False return self.thread_pool.submit_task(self.catch, *args, **kwargs) - def catch(self, *args, **kwargs): - return self.scan.manager.catch(*args, **kwargs) + async def catch(self, *args, **kwargs): + return await self.scan.manager.catch(*args, **kwargs) - def _postcheck_and_run(self, callback, event): + async def _postcheck_and_run(self, callback, event): acceptable, reason = self._event_postcheck(event) if not acceptable: if reason: self.debug(f"Not accepting {event} because {reason}") return self.scan.stats.event_consumed(event, self) - return callback(event) + return await callback(event) - def _register_running(self, callback, *args, **kwargs): - with self._running_counter: - return callback(*args, **kwargs) + async def _register_running(self, callback, *args, **kwargs): + with self._task_counter: + return await callback(*args, **kwargs) def _handle_batch(self, force=False): if self.batch_size <= 1: @@ -276,24 +269,7 @@ def emit_event(self, *args, **kwargs): for o in ("on_success_callback", "abort_if", "quick"): event_kwargs.pop(o, None) event = self.make_event(*args, **event_kwargs) - if event is None: - return - # nerf event's priority if it's likely not to be in scope - if event.scope_distance > 0: - event_in_scope = self.scan.whitelisted(event) and not self.scan.blacklisted(event) - if not event_in_scope: - event.module_priority += event.scope_distance - if event: - # Wait for parent event to resolve (in case its scope distance changes) - while 1: - if self.scan.stopping: - return - resolved = event.source._resolved.wait(timeout=0.1) - if resolved: - # update event's scope distance based on its parent - event.scope_distance = event.source.scope_distance + 1 - break - self.scan.manager.incoming_event_queue.put((event, kwargs)) + self.scan.manager.queue_event(event) @property def events_waiting(self): @@ -324,16 +300,15 @@ def num_queued_events(self): return ret def start(self): - self.thread = threading.Thread(target=self._worker, daemon=True) - self.thread.start() + self.tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] - def _setup(self): + async def _setup(self): status_codes = {False: "hard-fail", None: "soft-fail", True: "success"} status = False self.debug(f"Setting up module {self.name}") try: - result = self.setup() + result = await self.setup() if type(result) == tuple and len(result) == 2: status, msg = result else: @@ -346,7 +321,7 @@ def _setup(self): status = None msg = f"{e}" self.trace() - return status, str(msg) + return self.name, status, str(msg) @property def _force_batch(self): @@ -356,41 +331,36 @@ def _force_batch(self): # if we're below our maximum threading potential return self._internal_thread_pool.num_tasks < self.max_event_handlers - def _worker(self): + async def _worker(self): try: while not self.scan.stopping: # hold the reigns if our outgoing queue is full - if self._qsize and self.outgoing_event_queue_qsize >= self._qsize: - with self.event_received: - self.event_received.wait(timeout=0.1) - continue + # if self._qsize and self.outgoing_event_queue.qsize() >= self._qsize: + # with self.event_received: + # await self.event_received.wait() if self.batch_size > 1: - submitted = self._handle_batch(force=self._force_batch) - if not submitted: - with self.event_received: - self.event_received.wait(timeout=0.1) + pass + # submitted = self._handle_batch(force=self._force_batch) + # if not submitted: + # with self.event_received: + # await self.event_received.wait() else: try: if self.incoming_event_queue: - e = self.incoming_event_queue.get(timeout=0.1) + e = await self.incoming_event_queue.get() else: self.debug(f"Event queue is in bad state") return except queue.Empty: continue - self.debug(f"Got {e} from {getattr(e, 'module', e)}") + self.debug(f"Got {e} from {getattr(e, 'module', 'unknown_module')}") # if we receive the special "FINISHED" event if e.type == "FINISHED": - self._internal_thread_pool.submit_task(self.catch, self._register_running, self.finish) + await self.catch(self._register_running, self.finish) else: - if self._type == "output": - self.catch(self._register_running, self._postcheck_and_run, self.handle_event, e) - else: - self._internal_thread_pool.submit_task( - self.catch, self._register_running, self._postcheck_and_run, self.handle_event, e - ) + await self.catch(self._register_running, self._postcheck_and_run, self.handle_event, e) except KeyboardInterrupt: self.debug(f"Interrupted") @@ -486,12 +456,12 @@ def _event_postcheck(self, event): return True, "" - def _cleanup(self): + async def _cleanup(self): if not self._cleanedup: self._cleanedup = True for callback in [self.cleanup] + self.cleanup_callbacks: if callable(callback): - self.catch(self._register_running, callback, _force=True) + await self.catch(self._register_running, callback, _force=True) def queue_event(self, event): if self.incoming_event_queue in (None, False): @@ -503,11 +473,9 @@ def queue_event(self, event): self.debug(f"Not accepting {event} because {reason}") return try: - self.incoming_event_queue.put(event) + self.incoming_event_queue.put_nowait(event) except AttributeError: self.debug(f"Not in an acceptable state to queue event") - with self.event_received: - self.event_received.notify() def set_error_state(self, message=None): if not self.errored: @@ -535,34 +503,27 @@ def helpers(self): @property def status(self): - main_pool = self.thread_pool.num_tasks - internal_pool = self._internal_thread_pool.num_tasks - pool_total = main_pool + internal_pool - incoming_qsize = 0 - if self.incoming_event_queue: - incoming_qsize = self.incoming_event_queue.qsize() status = { - "events": {"incoming": incoming_qsize, "outgoing": self.outgoing_event_queue_qsize}, - "tasks": {"main_pool": main_pool, "internal_pool": internal_pool, "total": pool_total}, + "events": {"incoming": self.num_queued_events, "outgoing": self.outgoing_event_queue.qsize()}, + "tasks": self._task_counter.value, "errored": self.errored, } status["running"] = self.running - status["active"] = self._is_active(status) return status - @staticmethod - def _is_active(status): - if status["running"]: - return True - total = status["tasks"]["total"] + status["events"]["incoming"] + status["events"]["outgoing"] - return total > 0 - @property def running(self): """ Indicates whether the module is currently processing data. """ - return self._running_counter.value > 0 + return self._task_counter.value > 0 + + @property + def finished(self): + """ + Indicates whether the module is finished (not running and nothing in queues) + """ + return not self.running and self.num_queued_events <= 0 and self.outgoing_event_queue.qsize() <= 0 def request_with_fail_count(self, *args, **kwargs): r = self.helpers.request(*args, **kwargs) @@ -593,12 +554,14 @@ def config(self): @property def incoming_event_queue(self): if self._incoming_event_queue is None: - self._incoming_event_queue = queue.PriorityQueue() + self._incoming_event_queue = asyncio.PriorityQueue() return self._incoming_event_queue @property - def outgoing_event_queue_qsize(self): - return self.scan.manager.incoming_event_queue.modules.get(str(self), 0) + def outgoing_event_queue(self): + if self._outgoing_event_queue is None: + self._outgoing_event_queue = asyncio.PriorityQueue() + return self._outgoing_event_queue @property def priority(self): @@ -627,18 +590,17 @@ def __str__(self): return self.name def log_table(self, *args, **kwargs): - with self._report_lock: - table_name = kwargs.pop("table_name", None) - table = self.helpers.make_table(*args, **kwargs) - for line in table.splitlines(): - self.info(line) - if table_name is not None: - date = self.helpers.make_date() - filename = self.scan.home / f"{self.helpers.tagify(table_name)}-table-{date}.txt" - with open(filename, "w") as f: - f.write(table) - self.verbose(f"Wrote {table_name} to {filename}") - return table + table_name = kwargs.pop("table_name", None) + table = self.helpers.make_table(*args, **kwargs) + for line in table.splitlines(): + self.info(line) + if table_name is not None: + date = self.helpers.make_date() + filename = self.scan.home / f"{self.helpers.tagify(table_name)}-table-{date}.txt" + with open(filename, "w") as f: + f.write(table) + self.verbose(f"Wrote {table_name} to {filename}") + return table def stdout(self, *args, **kwargs): self.log.stdout(*args, extra={"scan_id": self.scan.id}, **kwargs) diff --git a/bbot/modules/internal/aggregate.py b/bbot/modules/internal/aggregate.py index ddda0b2b3e..b1f11b04e7 100644 --- a/bbot/modules/internal/aggregate.py +++ b/bbot/modules/internal/aggregate.py @@ -5,5 +5,5 @@ class aggregate(BaseReportModule): flags = ["passive", "safe"] meta = {"description": "Summarize statistics at the end of a scan"} - def report(self): + async def report(self): self.log_table(*self.scan.stats._make_table(), table_name="scan-stats") diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c57d6ffa10..6bf98a8604 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -299,7 +299,7 @@ class excavate(BaseInternalModule): deps_pip = ["pyjwt~=2.6.0"] - def setup(self): + async def setup(self): self.hostname = HostnameExtractor(self) self.url = URLExtractor(self) self.email = EmailExtractor(self) @@ -316,7 +316,7 @@ def search(self, source, extractors, event, **kwargs): for e in extractors: e.search(source, event, **kwargs) - def handle_event(self, event): + async def handle_event(self, event): data = event.data # HTTP_RESPONSE is a special case diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index f4f4724268..74e67ef507 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -25,7 +25,7 @@ class speculate(BaseInternalModule): _scope_shepherding = False _priority = 4 - def setup(self): + async def setup(self): self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()]) self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()]) self.range_to_ip = True @@ -47,7 +47,7 @@ def setup(self): return True - def handle_event(self, event): + async def handle_event(self, event): # generate individual IP addresses from IP range if event.type == "IP_RANGE" and self.range_to_ip: net = ipaddress.ip_network(event.data) diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index f4c0e235ad..f904728556 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -33,7 +33,7 @@ class asset_inventory(CSV): header_row = ["Host", "Provider", "IP(s)", "Status", "Open Ports", "Risk Rating", "Findings", "Description"] filename = "asset-inventory.csv" - def setup(self): + async def setup(self): self.assets = {} self.open_port_producers = "httpx" in self.scan.modules or any( ["portscan" in m.flags for m in self.scan.modules.values()] @@ -41,7 +41,7 @@ def setup(self): self.use_previous = self.config.get("use_previous", False) self.summary_netmask = self.config.get("summary_netmask", 16) self.emitted_contents = False - ret = super().setup() + ret = await super().setup() return ret def filter_event(self, event): diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index 866cb21497..feed0491db 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -13,7 +13,7 @@ class CSV(BaseOutputModule): header_row = ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"] filename = "output.csv" - def setup(self): + async def setup(self): self.custom_headers = [] self._headers_set = set() self._writer = None @@ -43,7 +43,7 @@ def writerow(self, row): self.writer.writerow(row) self.file.flush() - def handle_event(self, event): + async def handle_event(self, event): # ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"] self.writerow( { @@ -58,15 +58,14 @@ def handle_event(self, event): } ) - def cleanup(self): + async def cleanup(self): if getattr(self, "_file", None) is not None: with suppress(Exception): self.file.close() - def report(self): + async def report(self): if self._file is not None: - with self._report_lock: - self.info(f"Saved CSV output to {self.output_file}") + self.info(f"Saved CSV output to {self.output_file}") def add_custom_headers(self, headers): if isinstance(headers, str): diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 04c8e30101..1e3449df3b 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -25,7 +25,7 @@ class HTTP(BaseOutputModule): "timeout": "HTTP timeout", } - def setup(self): + async def setup(self): self.session = requests.Session() if not self.config.get("url", ""): self.warning("Must set URL") diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py index de9aa858d9..ac80144bb1 100644 --- a/bbot/modules/output/human.py +++ b/bbot/modules/output/human.py @@ -11,11 +11,11 @@ class Human(BaseOutputModule): options_desc = {"output_file": "Output to file", "console": "Output to console"} vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"} - def setup(self): + async def setup(self): self._prep_output_dir("output.txt") return True - def handle_event(self, event): + async def handle_event(self, event): event_type = f"[{event.type}]" event_tags = "" if getattr(event, "tags", []): @@ -36,12 +36,11 @@ def handle_event(self, event): if self.config.get("console", True): self.stdout(event_str) - def cleanup(self): + async def cleanup(self): if getattr(self, "_file", None) is not None: with suppress(Exception): self.file.close() - def report(self): + async def report(self): if self._file is not None: - with self._report_lock: - self.info(f"Saved TXT output to {self.output_file}") + self.info(f"Saved TXT output to {self.output_file}") diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index 6b1a698bf7..561354c45c 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -10,11 +10,11 @@ class JSON(BaseOutputModule): options = {"output_file": "", "console": False} options_desc = {"output_file": "Output to file", "console": "Output to console"} - def setup(self): + async def setup(self): self._prep_output_dir("output.json") return True - def handle_event(self, event): + async def handle_event(self, event): event_str = json.dumps(dict(event)) if self.file is not None: self.file.write(event_str + "\n") @@ -22,12 +22,11 @@ def handle_event(self, event): if self.config.get("console", False) or "human" not in self.scan.modules: self.stdout(event_str) - def cleanup(self): + async def cleanup(self): if getattr(self, "_file", None) is not None: with suppress(Exception): self.file.close() - def report(self): + async def report(self): if self._file is not None: - with self._report_lock: - self.info(f"Saved JSON output to {self.output_file}") + self.info(f"Saved JSON output to {self.output_file}") diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 55c00acb92..ea7e3ff720 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -19,7 +19,7 @@ class neo4j(BaseOutputModule): deps_pip = ["py2neo~=2021.2.3"] batch_size = 50 - def setup(self): + async def setup(self): try: self.neo4j = Neo4j( uri=self.config.get("uri", self.options["uri"]), diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index 40e8e8e483..da973a6e38 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -13,7 +13,7 @@ class web_report(BaseOutputModule): options_desc = {"output_file": "Output to file", "css_theme_file": "CSS theme URL for HTML output"} deps_pip = ["markdown~=3.4.3"] - def setup(self): + async def setup(self): html_css_file = self.config.get("css_theme_file", "") self.html_header = f""" diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index 2a7bc625ed..55b06f7adb 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -12,7 +12,7 @@ class Websocket(BaseOutputModule): options = {"url": "", "token": ""} options_desc = {"url": "Web URL", "token": "Authorization Bearer token"} - def setup(self): + async def setup(self): self.url = self.config.get("url", "") if not self.url: return False, "Must set URL" diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 04c36db881..331fddafd1 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -45,7 +45,7 @@ def handle_event(self, event): for email in emails: self.emit_event(email, "EMAIL_ADDRESS", source=asn_event) - def report(self): + async def report(self): asn_data = sorted(self.asn_cache.items(), key=lambda x: self.asn_counts[x[0]], reverse=True) if not asn_data: return diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index fd7d585daf..32620faa0d 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -1,12 +1,13 @@ import queue +import asyncio import logging -import threading import traceback from time import sleep from contextlib import suppress from datetime import datetime, timedelta from ..core.helpers.queueing import EventQueue +from ..core.helpers.async_helpers import TaskCounter from ..core.errors import ScanCancelledError, ValidationError log = logging.getLogger("bbot.scanner.manager") @@ -23,77 +24,74 @@ def __init__(self, scan): # tracks duplicate events on a global basis self.events_distributed = set() - # tracks duplicate events on a per-module basis self.events_accepted = set() - self.events_accepted_lock = threading.Lock() - - self._lock = threading.Lock() - self.event_emitted = threading.Condition(self._lock) - - self.events_resolved = dict() self.dns_resolution = self.scan.config.get("dns_resolution", False) - - self.status_frequency = self.scan.config.get("status_frequency", 15) + self._task_counter = TaskCounter() + self._new_activity = True self.last_log_time = datetime.now() - def init_events(self): + async def init_events(self): """ seed scanner with target events """ - self.distribute_event(self.scan.root_event) - sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) - for event in sorted_events: - self.scan.verbose(f"Target: {event}") - self.emit_event(event, _block=False, _force_submit=True) - # force submit batches - for mod in self.scan.modules.values(): - mod._handle_batch(force=True) - - def emit_event(self, event, *args, **kwargs): + with self._task_counter: + self.distribute_event(self.scan.root_event) + sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) + for event in sorted_events: + self.scan.verbose(f"Target: {event}") + self.queue_event(event) + # force submit batches + # for mod in self.scan.modules.values(): + # mod._handle_batch(force=True) + self.scan._finished_init = True + + async def emit_event(self, event, *args, **kwargs): """ TODO: Register + kill duplicate events immediately? bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) """ - # skip event if it fails precheck - if not self._event_precheck(event): - event._resolved.set() - return False - - # "quick" queues the event immediately - quick = kwargs.pop("quick", False) - if quick: - log.debug(f'Module "{event.module}" raised {event}') - event._resolved.set() - for kwarg in ["abort_if", "on_success_callback", "_block"]: - kwargs.pop(kwarg, None) - try: - self.distribute_event(event, *args, **kwargs) - return True - except ScanCancelledError: + with self._task_counter: + # skip event if it fails precheck + if not self._event_precheck(event): + event._resolved.set() return False - except Exception as e: - log.error(f"Unexpected error in manager.emit_event(): {e}") - log.trace(traceback.format_exc()) - else: - # don't raise an exception if the thread pool has been shutdown - try: - self.scan._event_thread_pool.submit_task(self.catch, self._emit_event, event, *args, **kwargs) + + # "quick" queues the event immediately + quick = kwargs.pop("quick", False) + if quick: log.debug(f'Module "{event.module}" raised {event}') - return True - except ScanCancelledError: - return False - except queue.Full: - raise - except Exception as e: - log.error(f"Unexpected error in manager.emit_event(): {e}") - log.trace(traceback.format_exc()) - finally: event._resolved.set() - return False + for kwarg in ["abort_if", "on_success_callback", "_block"]: + kwargs.pop(kwarg, None) + try: + self.distribute_event(event, *args, **kwargs) + return True + except ScanCancelledError: + return False + except Exception as e: + log.error(f"Unexpected error in manager.emit_event(): {e}") + log.trace(traceback.format_exc()) + else: + # don't raise an exception if the thread pool has been shutdown + error = True + try: + await self.catch(self._emit_event, event, *args, **kwargs) + error = False + log.debug(f'Module "{event.module}" raised {event}') + return True + except ScanCancelledError: + return False + except Exception as e: + log.error(f"Unexpected error in manager.emit_event(): {e}") + log.trace(traceback.format_exc()) + finally: + if error: + event._resolved.set() + return False def _event_precheck(self, event, exclude=("DNS_NAME",)): """ @@ -110,7 +108,7 @@ def _event_precheck(self, event, exclude=("DNS_NAME",)): return False return True - def _emit_event(self, event, *args, **kwargs): + async def _emit_event(self, event, *args, **kwargs): log.debug(f"Emitting {event}") distribute_event = True event_distributed = False @@ -134,7 +132,7 @@ def _emit_event(self, event, *args, **kwargs): event_whitelisted_dns, event_blacklisted_dns, dns_children, - ) = self.scan.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) + ) = await self.scan.helpers.dns.resolve_event(event, minimal=not self.dns_resolution) resolved_hosts = set() for rdtype, ips in dns_children.items(): if rdtype in ("A", "AAAA", "CNAME"): @@ -193,7 +191,7 @@ def _emit_event(self, event, *args, **kwargs): log.debug(f"Making {event} in-scope") source_trail = event.make_in_scope(set_scope_distance) for s in source_trail: - self.emit_event(s, _block=False, _force_submit=True) + self.queue_event(s) else: if event.scope_distance > self.scan.scope_report_distance: log.debug( @@ -205,7 +203,7 @@ def _emit_event(self, event, *args, **kwargs): if event.scope_distance <= self.scan.scope_search_distance: if not "unresolved" in event.tags: if not self.scan.helpers.is_ip_type(event.host): - self.scan.helpers.dns.handle_wildcard_event(event, dns_children) + await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) # now that the event is properly tagged, we can finally make decisions about it if callable(abort_if): @@ -232,7 +230,7 @@ def _emit_event(self, event, *args, **kwargs): ) source_trail = event.unmake_internal(force_output=True) for s in source_trail: - self.emit_event(s, _block=False, _force_submit=True) + self.queue_event(s) if distribute_event: self.distribute_event(event) @@ -253,17 +251,16 @@ def _emit_event(self, event, *args, **kwargs): source_event.scope_distance = event.scope_distance if "target" in event.tags: source_event.add_tag("target") - self.emit_event(source_event, _block=False, _force_submit=True) + self.queue_event(source_event) ### Emit DNS children ### if self.dns_resolution: emit_children = -1 < event.scope_distance < self.scan.dns_search_distance if emit_children: host_hash = hash(str(event.host)) - with self.events_accepted_lock: - if host_hash in self.events_accepted: - emit_children = False - self.events_accepted.add(host_hash) + if host_hash in self.events_accepted: + emit_children = False + self.events_accepted.add(host_hash) if emit_children: dns_child_events = [] @@ -282,7 +279,7 @@ def _emit_event(self, event, *args, **kwargs): f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' ) for child_event in dns_child_events: - self.emit_event(child_event, _block=False, _force_submit=True) + self.queue_event(child_event) except ValidationError as e: log.warning(f"Event validation failed with args={args}, kwargs={kwargs}: {e}") @@ -292,8 +289,6 @@ def _emit_event(self, event, *args, **kwargs): event._resolved.set() if event_distributed: self.scan.stats.event_distributed(event) - with self.event_emitted: - self.event_emitted.notify() log.debug(f"{event.module}.emit_event() finished for {event}") def hash_event(self, event): @@ -316,10 +311,9 @@ def is_duplicate_event(self, event, add=False): """ event_hash = self.hash_event(event) suppress_dupes = getattr(event.module, "suppress_dupes", True) - with self.events_accepted_lock: - duplicate_event = suppress_dupes and event_hash in self.events_accepted - if add: - self.events_accepted.add(event_hash) + duplicate_event = suppress_dupes and event_hash in self.events_accepted + if add: + self.events_accepted.add(event_hash) return duplicate_event def accept_event(self, event): @@ -329,7 +323,7 @@ def accept_event(self, event): return False return True - def catch(self, callback, *args, **kwargs): + async def catch(self, callback, *args, **kwargs): """ Wrapper to ensure error messages get surfaced to the user """ @@ -344,7 +338,10 @@ def catch(self, callback, *args, **kwargs): break try: if not self.scan.stopping or force: - ret = callback(*args, **kwargs) + if self.scan.helpers.is_async_function(callback): + ret = await callback(*args, **kwargs) + else: + ret = callback(*args, **kwargs) except ScanCancelledError as e: log.debug(f"ScanCancelledError in {fn.__qualname__}(): {e}") except BrokenPipeError as e: @@ -357,7 +354,10 @@ def catch(self, callback, *args, **kwargs): self.scan.stop() if callable(on_finish_callback): try: - on_finish_callback() + if self.scan.helpers.is_async_function(on_finish_callback): + await on_finish_callback() + else: + on_finish_callback() except Exception as e: log.error( f"Error in on_finish_callback {on_finish_callback.__qualname__}() after {fn.__qualname__}(): {e}" @@ -365,6 +365,10 @@ def catch(self, callback, *args, **kwargs): log.trace(traceback.format_exc()) return ret + async def _register_running(self, callback, *args, **kwargs): + with self._task_counter: + return await callback(*args, **kwargs) + def distribute_event(self, *args, **kwargs): """ Queue event with modules @@ -384,63 +388,16 @@ def distribute_event(self, *args, **kwargs): if not dup or mod.accept_dupes: mod.queue_event(event) - def loop_until_finished(self): - modules = list(self.scan.modules.values()) - activity = True - + async def _worker_loop(self): try: - self.scan.dispatcher.on_start(self.scan) - while 1: - # abort if we're aborting - if self.scan.aborting: - # Empty event queues - for module in self.scan.modules.values(): - with suppress(queue.Empty): - while 1: - module.incoming_event_queue.get_nowait() - with suppress(queue.Empty): - while 1: - self.incoming_event_queue.get_nowait() - break - - if "python" in self.scan.modules: - events, finish, report = self.scan.modules["python"].events_waiting - yield from events - try: - self.log_status(self.status_frequency) event, kwargs = self.incoming_event_queue.get_nowait() - while not self.scan.aborting: - try: - acceptable = self.emit_event(event, _block=False, **kwargs) - if acceptable: - activity = True - break - except queue.Full: - self.log_status(self.status_frequency) - with self.event_emitted: - self.event_emitted.wait(timeout=0.1) + acceptable = await self.emit_event(event, **kwargs) + if acceptable: + self._new_activity = True except queue.Empty: - # if we're on the last module - modules_status = self.modules_status() - finished = modules_status.get("finished", False) - # And if the scan is finished - if finished: - # And if new events were generated since last time we were here - if activity: - activity = False - self.scan.status = "FINISHING" - # Trigger .finished() on every module and start over - log.info("Finishing scan") - finished_event = self.scan.make_event("FINISHED", "FINISHED", dummy=True) - for module in modules: - module.queue_event(finished_event) - else: - # Otherwise stop the scan if no new events were generated since last time - break - with self.incoming_event_queue.not_empty: - self.incoming_event_queue.not_empty.wait(timeout=0.1) + await asyncio.sleep(0.1) except KeyboardInterrupt: self.scan.stop() @@ -448,52 +405,45 @@ def loop_until_finished(self): except Exception: log.critical(traceback.format_exc()) - finally: - # Run .report() on every module - for mod in self.scan.modules.values(): - self.catch(mod._register_running, mod.report, _force=True) - - def log_status(self, frequency=15): - # print status every 15 seconds (or status_frequency setting) - timedelta_secs = timedelta(seconds=frequency) - now = datetime.now() - time_since_last_log = now - self.last_log_time - if time_since_last_log > timedelta_secs: - self.modules_status(_log=True, passes=1) - self.last_log_time = now - - def modules_status(self, _log=False, passes=None): - # If scan looks to be finished, check an additional five times to ensure that it really is - # There is a tiny chance of a race condition, which this helps to avoid - if passes is None: - passes = 5 - else: - passes = max(1, int(passes)) - + def queue_event(self, event, **kwargs): + if event: + # nerf event's priority if it's likely not to be in scope + if event.scope_distance > 0: + event_in_scope = self.scan.whitelisted(event) and not self.scan.blacklisted(event) + if not event_in_scope: + event.module_priority += event.scope_distance + # Wait for parent event to resolve (in case its scope distance changes) + # await resolved = event.source._resolved.wait() + # update event's scope distance based on its parent + event.scope_distance = event.source.scope_distance + 1 + self.incoming_event_queue.put_nowait((event, kwargs)) + + @property + def running(self): + return self._task_counter.value > 0 or self.incoming_event_queue.qsize() > 0 + + @property + def modules_finished(self): + return all(m.finished for m in self.scan.modules.values()) + + @property + def active(self): + return self.running or not self.modules_finished + + async def modules_status(self, _log=False): finished = True - while passes > 0: - status = {"modules": {}, "scan": self.scan.status_detailed} - - for num_tasks in status["scan"]["queued_tasks"].values(): - if num_tasks > 0: - finished = False - - for m in self.scan.modules.values(): - mod_status = m.status - if mod_status["active"]: - finished = False - status["modules"][m.name] = mod_status - - for mod in self.scan.modules.values(): - if mod.errored and mod.incoming_event_queue not in [None, False]: - with suppress(Exception): - mod.set_error_state() - - passes -= 1 - if finished and passes > 0: - sleep(0.1) - else: - break + status = {"modules": {}} + + for m in self.scan.modules.values(): + mod_status = m.status + if mod_status["running"]: + finished = False + status["modules"][m.name] = mod_status + + for mod in self.scan.modules.values(): + if mod.errored and mod.incoming_event_queue not in [None, False]: + with suppress(Exception): + mod.set_error_state() status["finished"] = finished @@ -505,7 +455,7 @@ def modules_status(self, _log=False, passes=None): running = s["running"] incoming = s["events"]["incoming"] outgoing = s["events"]["outgoing"] - tasks = s["tasks"]["total"] + tasks = s["tasks"] total = sum([incoming, outgoing, tasks]) if running or total > 0: modules_status.append((m, running, incoming, outgoing, tasks, total)) @@ -528,13 +478,6 @@ def modules_status(self, _log=False, passes=None): else: self.scan.info(f"{self.scan.name}: No events produced yet") - total_tasks = status["scan"]["queued_tasks"]["total"] - event_tasks = status["scan"]["queued_tasks"]["event"] - internal_tasks = status["scan"]["queued_tasks"]["internal"] - self.scan.verbose( - f"{self.scan.name}: Thread pool tasks: {total_tasks:,} (Event: {event_tasks:,}, Internal: {internal_tasks:,})" - ) - if modules_errored: self.scan.verbose( f'{self.scan.name}: Modules errored: {len(modules_errored):,} ({", ".join([m for m in modules_errored])})' @@ -550,29 +493,6 @@ def modules_status(self, _log=False, passes=None): else: self.scan.info(f"{self.scan.name}: No events in queue") - # if debugging is enabled - self.scan.debug(f"THREAD POOL STATUS:") - if self.scan.log_level <= logging.DEBUG: - # log thread pool statuses - threadpool_names = [ - "_internal_thread_pool", - "_event_thread_pool", - "_thread_pool", - ] - for threadpool_name in threadpool_names: - threadpool = getattr(self.scan, threadpool_name) - for thread_status in threadpool.threads_status: - self.scan.debug(f" - {threadpool_name}: {thread_status}") - # log module memory usage - module_memory_usage = [] - for module in self.scan.modules.values(): - memory_usage = module.memory_usage - module_memory_usage.append((module.name, memory_usage)) - module_memory_usage.sort(key=lambda x: x[-1], reverse=True) - self.scan.debug(f"MODULE MEMORY USAGE:") - for module_name, usage in module_memory_usage: - self.scan.debug(f" - {module_name}: {self.scan.helpers.bytes_to_human(usage)}") - # Uncomment these lines to enable debugging of event queues # queued_events = self.incoming_event_queue.events diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 01a7e0831a..0b33903727 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1,5 +1,5 @@ +import asyncio import logging -import threading import traceback from sys import exc_info from pathlib import Path @@ -22,7 +22,6 @@ from bbot.core.logger import init_logging, get_log_level from bbot.core.helpers.names_generator import random_name from bbot.core.configurator.environ import prepare_environment -from bbot.core.helpers.threadpool import ThreadPoolWrapper, BBOTThreadPoolExecutor from bbot.core.errors import BBOTError, ScanError, ScanCancelledError, ValidationError log = logging.getLogger("bbot.scanner") @@ -84,25 +83,8 @@ def __init__( self._status = "NOT_STARTED" self._status_code = 0 - # Set up thread pools - max_workers = max(1, self.config.get("max_threads", 25)) - # Shared thread pool, for module use - self._thread_pool = BBOTThreadPoolExecutor(max_workers=max_workers) - # Event thread pool, for event emission - self._event_thread_pool = ThreadPoolWrapper( - BBOTThreadPoolExecutor(max_workers=max_workers * 2), qsize=max_workers - ) - # Internal thread pool, for handle_event(), module setup, cleanup callbacks, etc. - self._internal_thread_pool = ThreadPoolWrapper(BBOTThreadPoolExecutor(max_workers=max_workers)) - self.process_pool = ThreadPoolWrapper(concurrent.futures.ProcessPoolExecutor()) + self.max_workers = max(1, self.config.get("max_threads", 25)) self.helpers = ConfigAwareHelper(config=self.config, scan=self) - self.pools = { - "process_pool": self.process_pool, - "internal_thread_pool": self._internal_thread_pool, - "dns_thread_pool": self.helpers.dns._thread_pool, - "event_thread_pool": self._event_thread_pool, - "main_thread_pool": self._thread_pool, - } output_dir = self.config.get("output_dir", "") if name is None: @@ -170,12 +152,14 @@ def __init__( "You have enabled custom HTTP headers. These will be attached to all in-scope requests and all requests made by httpx." ) + # how often to print scan status + self.status_frequency = self.config.get("status_frequency", 15) + self._prepped = False - self._thread_pools_shutdown = False - self._thread_pools_shutdown_threads = [] + self._finished_init = False self._cleanedup = False - def prep(self): + async def prep(self): self.helpers.mkdir(self.home) if not self._prepped: start_msg = f"Scan with {len(self._scan_modules):,} modules seeded with {len(self.target):,} targets" @@ -191,22 +175,26 @@ def prep(self): self.load_modules() self.info(f"Setting up modules...") - self.setup_modules() + await self.setup_modules() self.success(f"Setup succeeded for {len(self.modules):,} modules.") self._prepped = True - def start_without_generator(self): - deque(self.start(), maxlen=0) + async def start_without_generator(self): + async for event in self.start(): + pass - def start(self): - self.prep() + async def start(self): + await self.prep() failed = True if not self.target: self.warning(f"No scan targets specified") + # start status ticker + ticker_task = asyncio.create_task(self._status_ticker(self.)) + scan_start_time = datetime.now() try: self.status = "STARTING" @@ -218,23 +206,59 @@ def start(self): else: self.hugesuccess(f"Starting scan {self.name}") - if self.stopping: - return + self.dispatcher.on_start(self) - # distribute seed events - self.manager.init_events() + # start manager worker loops + manager_worker_loop_tasks = [ + asyncio.create_task(self.manager._worker_loop()) for _ in range(self.max_workers) + ] - if self.stopping: - return + # distribute seed events + init_events_task = asyncio.create_task(self.manager.init_events()) self.status = "RUNNING" self.start_modules() self.verbose(f"{len(self.modules):,} modules started") - if self.stopping: - return + # main scan loop + while 1: + # abort if we're aborting + if self.aborting: + # Empty event queues + for module in self.modules.values(): + with suppress(queue.Empty): + while 1: + module.incoming_event_queue.get_nowait() + with suppress(queue.Empty): + while 1: + self.incoming_event_queue.get_nowait() + break + + if "python" in self.modules: + events, finish, report = self.modules["python"].events_waiting + for e in events: + yield e + + if self._finished_init and not self.manager.active: + # And if new events were generated since last time we were here + if self.manager._new_activity: + self.manager._new_activity = False + self.status = "FINISHING" + # Trigger .finished() on every module and start over + log.info("Finishing scan") + finished_event = self.make_event("FINISHED", "FINISHED", dummy=True) + for module in self.modules.values(): + module.queue_event(finished_event) + else: + # Otherwise stop the scan if no new events were generated since last time + break + + await asyncio.sleep(0.01) + + # for module in self.modules.values(): + # for task in module.tasks: + # await task - yield from self.manager.loop_until_finished() failed = False except KeyboardInterrupt: @@ -255,21 +279,8 @@ def start(self): self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") finally: - self.cleanup() - self.shutdown_threadpools() - while 1: - for t in self._thread_pools_shutdown_threads: - t.join(timeout=1) - if t.is_alive(): - try: - pool = t._args[0] - for s in pool.threads_status: - self.debug(s) - except AttributeError: - continue - if not any(t.is_alive() for t in self._thread_pools_shutdown_threads): - self.debug("Finished shutting down thread pools") - break + await self.report() + await self.cleanup() log_fn = self.hugesuccess if self.status == "ABORTING": @@ -281,6 +292,19 @@ def start(self): else: self.status = "FINISHED" + ticker_task.cancel() + with suppress(asyncio.CancelledError): + await ticker_task + + init_events_task.cancel() + with suppress(asyncio.CancelledError): + await init_events_task + + for t in manager_worker_loop_tasks: + t.cancel() + with suppress(asyncio.CancelledError): + await t + scan_run_time = datetime.now() - scan_start_time scan_run_time = self.helpers.human_timedelta(scan_run_time) log_fn(f"Scan {self.name} completed in {scan_run_time} with status {self.status}") @@ -288,23 +312,19 @@ def start(self): self.dispatcher.on_finish(self) def start_modules(self): - self.verbose(f"Starting module threads") + self.verbose(f"Starting module worker loops") for module_name, module in self.modules.items(): module.start() - def setup_modules(self, remove_failed=True): + async def setup_modules(self, remove_failed=True): self.load_modules() self.verbose(f"Setting up modules") hard_failed = [] soft_failed = [] setup_futures = dict() - for module_name, module in self.modules.items(): - future = self._internal_thread_pool.submit_task(module._setup) - setup_futures[future] = module_name - for future in self.helpers.as_completed(setup_futures): - module_name = setup_futures[future] - status, msg = future.result() + for task in asyncio.as_completed([m._setup() for m in self.modules.values()]): + module_name, status, msg = await task if status == True: self.debug(f"Setup succeeded for {module_name} ({msg})") elif status == False: @@ -332,34 +352,17 @@ def stop(self, wait=False): self.status = "ABORTING" self.hugewarning(f"Aborting scan") self.helpers.kill_children() - self.shutdown_threadpools() self.helpers.kill_children() - def shutdown_threadpools(self): - if not self._thread_pools_shutdown: - self._thread_pools_shutdown = True - - def shutdown_pool(pool, pool_name, **kwargs): - self.debug(f"Shutting down {pool_name} with kwargs={kwargs}") - pool.shutdown(**kwargs) - self.debug(f"Finished shutting down {pool_name} with kwargs={kwargs}") - - self.debug(f"Shutting down thread pools") - for pool_name, pool in self.pools.items(): - t = threading.Thread( - target=shutdown_pool, - args=(pool, pool_name), - kwargs={"wait": True, "cancel_futures": True}, - daemon=True, - ) - t.start() - self._thread_pools_shutdown_threads.append(t) + async def report(self): + for mod in self.modules.values(): + await self.manager.catch(mod._register_running, mod.report) - def cleanup(self): + async def cleanup(self): # clean up modules self.status = "CLEANING_UP" for mod in self.modules.values(): - mod._cleanup() + await mod._cleanup() if not self._cleanedup: self._cleanedup = True with suppress(Exception): @@ -393,6 +396,10 @@ def word_cloud(self): def stopping(self): return not self.running + @property + def stopped(self): + return self._status_code > 5 + @property def running(self): return 0 < self._status_code < 4 @@ -421,22 +428,6 @@ def status(self, status): else: self.debug(f'Attempt to set invalid status "{status}" on scan') - @property - def status_detailed(self): - event_threadpool_tasks = self._event_thread_pool.num_tasks - internal_tasks = self._internal_thread_pool.num_tasks - process_tasks = self.process_pool.num_tasks - total_tasks = event_threadpool_tasks + internal_tasks + process_tasks - status = { - "queued_tasks": { - "internal": internal_tasks, - "process": process_tasks, - "event": event_threadpool_tasks, - "total": total_tasks, - }, - } - return status - def make_event(self, *args, **kwargs): kwargs["scan"] = self event = make_event(*args, **kwargs) @@ -614,3 +605,8 @@ def _load_modules(self, modules): self.warning(f'Failed to load unknown module "{module_name}"') failed.add(module_name) return loaded_modules, failed + + async def _status_ticker(self, interval=15): + while not self.stopped: + await asyncio.sleep(interval) + await self.manager.modules_status(_log=True) diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 52ee009ba6..597d313fc4 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -29,7 +29,7 @@ scope_search_distance: 0 scope_report_distance: 0 scope_dns_search_distance: 1 plumbus: asdf -dns_debug: false +dns_debug: true http_debug: false keep_scans: 1 agent_url: test diff --git a/bbot/test/test_step_2/test_helpers.py b/bbot/test/test_step_2/test_helpers.py index d10273d4ee..5ea38e8ff6 100644 --- a/bbot/test/test_step_2/test_helpers.py +++ b/bbot/test/test_step_2/test_helpers.py @@ -483,113 +483,11 @@ def plumbus_generator(): helpers.wordlist("/tmp/a9pseoysadf/asdkgjaosidf") test_file.unlink() - ### DNS ### - # resolution - assert all([helpers.is_ip(i) for i in helpers.resolve("scanme.nmap.org")]) - assert "dns.google" in helpers.resolve("8.8.8.8") - assert "dns.google" in helpers.resolve("2001:4860:4860::8888") - resolved_ips = helpers.resolve("dns.google") - assert "2001:4860:4860::8888" in resolved_ips - assert "8.8.8.8" in resolved_ips - assert any([helpers.is_subdomain(h) for h in helpers.resolve("google.com", type="mx")]) - v6_ips = helpers.resolve("www.google.com", type="AAAA") - assert all([i.version == 6 for i in [ipaddress.ip_address(_) for _ in v6_ips]]) - assert not helpers.resolve(f"{helpers.rand_string(length=30)}.com") - # batch resolution - batch_results = list(helpers.resolve_batch(["8.8.8.8", "dns.google"])) - assert len(batch_results) == 2 - batch_results = dict(batch_results) - assert any([x in batch_results["dns.google"] for x in ("8.8.8.8", "8.8.4.4")]) - assert "dns.google" in batch_results["8.8.8.8"] - # "any" type - resolved = helpers.resolve("google.com", type="any") - assert any([helpers.is_subdomain(h) for h in resolved]) - # dns cache - assert hash(f"8.8.8.8:PTR") not in helpers.dns._dns_cache - assert hash(f"scanme.nmap.org:A") not in helpers.dns._dns_cache - assert hash(f"scanme.nmap.org:AAAA") not in helpers.dns._dns_cache - helpers.resolve("8.8.8.8", cache_result=True) - assert hash(f"8.8.8.8:PTR") in helpers.dns._dns_cache - helpers.resolve("scanme.nmap.org", cache_result=True) - assert hash(f"scanme.nmap.org:A") in helpers.dns._dns_cache - assert hash(f"scanme.nmap.org:AAAA") in helpers.dns._dns_cache - # wildcards - wildcard_domains = helpers.is_wildcard_domain("asdf.github.io") - assert "github.io" in wildcard_domains - assert "A" in wildcard_domains["github.io"] - assert "SRV" not in wildcard_domains["github.io"] - assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"]) - wildcard_rdtypes = helpers.is_wildcard("blacklanternsecurity.github.io") - assert "A" in wildcard_rdtypes - assert "SRV" not in wildcard_rdtypes - assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in helpers.dns._wildcard_cache - assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 - helpers.dns._wildcard_cache.clear() - wildcard_rdtypes = helpers.is_wildcard("asdf.asdf.asdf.github.io") - assert "A" in wildcard_rdtypes - assert "SRV" not in wildcard_rdtypes - assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in helpers.dns._wildcard_cache - assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 - wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", dummy=True) - wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", dummy=True) - wildcard_event3 = scan.make_event("github.io", "DNS_NAME", dummy=True) - event_tags1, event_whitelisted1, event_blacklisted1, children1 = scan.helpers.resolve_event(wildcard_event1) - event_tags2, event_whitelisted2, event_blacklisted2, children2 = scan.helpers.resolve_event(wildcard_event2) - event_tags3, event_whitelisted3, event_blacklisted3, children3 = scan.helpers.resolve_event(wildcard_event3) - helpers.handle_wildcard_event(wildcard_event1, children1) - helpers.handle_wildcard_event(wildcard_event2, children2) - helpers.handle_wildcard_event(wildcard_event3, children3) - assert "wildcard" in wildcard_event1.tags - assert "a-wildcard" in wildcard_event1.tags - assert "srv-wildcard" not in wildcard_event1.tags - assert "wildcard" in wildcard_event2.tags - assert "a-wildcard" in wildcard_event2.tags - assert "srv-wildcard" not in wildcard_event2.tags - assert wildcard_event1.data == "_wildcard.github.io" - assert wildcard_event2.data == "_wildcard.github.io" - assert wildcard_event1.tags == wildcard_event2.tags - assert "wildcard-domain" in wildcard_event3.tags - assert "a-wildcard-domain" in wildcard_event3.tags - assert "srv-wildcard-domain" not in wildcard_event3.tags - # misc dns helpers + # misc DNS helpers assert helpers.is_ptr("wsc-11-22-33-44-wat.evilcorp.com") == True assert helpers.is_ptr("wsc-11-22-33-wat.evilcorp.com") == False assert helpers.is_ptr("11wat.evilcorp.com") == False - # Ensure events with hosts have resolved_hosts attribute populated - - resolved_hosts_event1 = scan.make_event("dns.google", "DNS_NAME", dummy=True) - resolved_hosts_event2 = scan.make_event("http://dns.google/", "URL_UNVERIFIED", dummy=True) - event_tags1, event_whitelisted1, event_blacklisted1, children1 = scan.helpers.resolve_event(resolved_hosts_event1) - event_tags2, event_whitelisted2, event_blacklisted2, children2 = scan.helpers.resolve_event(resolved_hosts_event2) - - assert "8.8.8.8" in [str(x) for x in children1["A"]] - assert "8.8.8.8" in [str(x) for x in children2["A"]] - assert set(children1.keys()) == set(children2.keys()) - - msg = "Ignore this error, it belongs here" - - def raise_e(): - raise Exception(msg) - - def raise_k(): - raise KeyboardInterrupt(msg) - - def raise_s(): - raise ScanCancelledError(msg) - - def raise_b(): - raise BrokenPipeError(msg) - - helpers.dns._catch_keyboardinterrupt(raise_e) - helpers.dns._catch_keyboardinterrupt(raise_k) - scan.manager.catch(raise_e, _on_finish_callback=raise_e) - scan.manager.catch(raise_k) - scan.manager.catch(raise_s) - scan.manager.catch(raise_b) - ## NTLM testheader = "TlRMTVNTUAACAAAAHgAeADgAAAAVgorilwL+bvnVipUAAAAAAAAAAJgAmABWAAAACgBjRQAAAA9XAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAACAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgAAQAeAFcASQBOAC0AUwA0ADIATgBPAEIARABWAFQASwA4AAQAHgBXAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAADAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgABwAIAHUwOZlfoNgBAAAAAA==" decoded = helpers.ntlm.ntlmdecode(testheader) diff --git a/bbot/test/test_step_2/test_manager.py b/bbot/test/test_step_2/test_manager.py index ec58276ec4..4a0bcff54a 100644 --- a/bbot/test/test_step_2/test_manager.py +++ b/bbot/test/test_step_2/test_manager.py @@ -87,6 +87,26 @@ class DummyModule3: assert len(event_children) == 0 assert googledns in output + # error catching + msg = "Ignore this error, it belongs here" + + def raise_e(): + raise Exception(msg) + + def raise_k(): + raise KeyboardInterrupt(msg) + + def raise_s(): + raise ScanCancelledError(msg) + + def raise_b(): + raise BrokenPipeError(msg) + + manager.catch(raise_e, _on_finish_callback=raise_e) + manager.catch(raise_k) + manager.catch(raise_s) + manager.catch(raise_b) + def test_scope_distance(bbot_scanner, bbot_config): # event filtering based on scope_distance diff --git a/poetry.lock b/poetry.lock index ec3eeedca2..d563198a5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1057,6 +1057,25 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.21.0" +description = "Pytest support for asyncio" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-asyncio-0.21.0.tar.gz", hash = "sha256:2b38a496aef56f56b0e87557ec313e11e1ab9276fc3863f6a7be0f1d0e415e1b"}, + {file = "pytest_asyncio-0.21.0-py3-none-any.whl", hash = "sha256:f2b3366b7cd501a4056858bd39349d5af19742aed2d81660b7998b6341c7eb9c"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] + [[package]] name = "pytest-cov" version = "4.0.0" @@ -1491,4 +1510,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5154e783b3e49545e75315e8ae36f21d279bd21b3e108ef4bdadd5f3c340f35a" +content-hash = "010faf1151a864dfd8f7d72f16be9457675f1359dcb30393ed2ca8db47b3887c" diff --git a/pyproject.toml b/pyproject.toml index 27b476decc..32b6df29b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ requests-mock = "^1.10.0" poetry-dynamic-versioning = "^0.21.4" pytest-httpserver = "^1.0.6" pytest-rerunfailures = "^11.1.2" +pytest-asyncio = "^0.21.0" [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] From 0bf518a5ce03967bdf0228a6e349a1ee2f50c02c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 5 May 2023 10:28:17 -0400 Subject: [PATCH 005/387] websocket --- bbot/modules/output/websocket.py | 68 ++++++++++++------------- bbot/scanner/manager.py | 3 ++ bbot/scanner/scanner.py | 8 +-- poetry.lock | 85 +++++++++++++++++++++++++++----- pyproject.toml | 2 +- 5 files changed, 117 insertions(+), 49 deletions(-) diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index 55b06f7adb..7e302d0b66 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -1,6 +1,6 @@ import json import threading -import websocket +import websockets from time import sleep from bbot.modules.output.base import BaseOutputModule @@ -16,43 +16,45 @@ async def setup(self): self.url = self.config.get("url", "") if not self.url: return False, "Must set URL" - kwargs = {} self.token = self.config.get("token", "") - if self.token: - kwargs.update({"header": {"Authorization": f"Bearer {self.token}"}}) - self.ws = websocket.WebSocketApp(self.url, **kwargs) - self.started = False + self._ws = None return True - def start_websocket(self): - if not self.started: - self.thread = threading.Thread(target=self._start_websocket, daemon=True) - self.thread.start() - self.started = True - - def _start_websocket(self): - not_keyboardinterrupt = False - while not self.scan.stopping: - not_keyboardinterrupt = self.ws.run_forever() - if not not_keyboardinterrupt: - break - sleep(1) - - def handle_event(self, event): - self.start_websocket() + async def handle_event(self, event): event_json = event.json() - self.send(event_json) - - def send(self, message): - while self.ws is not None: + await self.send(event_json) + + async def ws(self, rebuild=False): + if self._ws is None or rebuild: + kwargs = {"close_timeout": 0.5} + if self.token: + kwargs.update({"extra_headers": {"Authorization": f"Bearer {self.token}"}}) + verbs = ("Building", "Built") + if rebuild: + verbs = ("Rebuilding", "Rebuilt") + self.debug(f"{verbs[0]} websocket connection to {self.url}") + self._ws = await websockets.connect(self.url, **kwargs) + self.debug(f"{verbs[1]} websocket connection to {self.url}") + return self._ws + + async def send(self, message): + rebuild = False + while not self.scan.stopped: try: - self.ws.send(json.dumps(message)) + ws = await self.ws(rebuild=rebuild) + message = json.dumps(message) + self.debug(f"Sending message of length {len(message)}") + await ws.send(message) + rebuild = False break except Exception as e: self.warning(f"Error sending message: {e}, retrying") - sleep(1) - continue - - def cleanup(self): - self.ws.close() - self.ws = None + await asyncio.sleep(1) + rebuild = True + + async def cleanup(self): + if self._ws is not None: + self.debug(f"Closing connection to {self.url}") + await self._ws.close() + self.debug(f"Closed connection to {self.url}") + self._ws = None diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 32620faa0d..5a18cdac2f 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -281,6 +281,9 @@ async def _emit_event(self, event, *args, **kwargs): for child_event in dns_child_events: self.queue_event(child_event) + except KeyboardInterrupt: + self.scan.stop() + except ValidationError as e: log.warning(f"Event validation failed with args={args}, kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 0b33903727..11df8d770d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -279,6 +279,10 @@ async def start(self): self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") finally: + init_events_task.cancel() + with suppress(asyncio.CancelledError): + await init_events_task + await self.report() await self.cleanup() @@ -296,10 +300,6 @@ async def start(self): with suppress(asyncio.CancelledError): await ticker_task - init_events_task.cancel() - with suppress(asyncio.CancelledError): - await init_events_task - for t in manager_worker_loop_tasks: t.cancel() with suppress(asyncio.CancelledError): diff --git a/poetry.lock b/poetry.lock index d563198a5a..a032ccadeb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1435,22 +1435,85 @@ secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "p socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] -name = "websocket-client" -version = "1.5.1" -description = "WebSocket client for Python with low level API options" +name = "websockets" +version = "11.0.2" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "websocket-client-1.5.1.tar.gz", hash = "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40"}, - {file = "websocket_client-1.5.1-py3-none-any.whl", hash = "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"}, + {file = "websockets-11.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:580cc95c58118f8c39106be71e24d0b7e1ad11a155f40a2ee687f99b3e5e432e"}, + {file = "websockets-11.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:143782041e95b63083b02107f31cda999f392903ae331de1307441f3a4557d51"}, + {file = "websockets-11.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8df63dcd955eb6b2e371d95aacf8b7c535e482192cff1b6ce927d8f43fb4f552"}, + {file = "websockets-11.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9b2dced5cbbc5094678cc1ec62160f7b0fe4defd601cd28a36fde7ee71bbb5"}, + {file = "websockets-11.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0eeeea3b01c97fd3b5049a46c908823f68b59bf0e18d79b231d8d6764bc81ee"}, + {file = "websockets-11.0.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:502683c5dedfc94b9f0f6790efb26aa0591526e8403ad443dce922cd6c0ec83b"}, + {file = "websockets-11.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d3cc3e48b6c9f7df8c3798004b9c4b92abca09eeea5e1b0a39698f05b7a33b9d"}, + {file = "websockets-11.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:808b8a33c961bbd6d33c55908f7c137569b09ea7dd024bce969969aa04ecf07c"}, + {file = "websockets-11.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:34a6f8996964ccaa40da42ee36aa1572adcb1e213665e24aa2f1037da6080909"}, + {file = "websockets-11.0.2-cp310-cp310-win32.whl", hash = "sha256:8f24cd758cbe1607a91b720537685b64e4d39415649cac9177cd1257317cf30c"}, + {file = "websockets-11.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:3b87cd302f08ea9e74fdc080470eddbed1e165113c1823fb3ee6328bc40ca1d3"}, + {file = "websockets-11.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3565a8f8c7bdde7c29ebe46146bd191290413ee6f8e94cf350609720c075b0a1"}, + {file = "websockets-11.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f97e03d4d5a4f0dca739ea274be9092822f7430b77d25aa02da6775e490f6846"}, + {file = "websockets-11.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8f392587eb2767afa8a34e909f2fec779f90b630622adc95d8b5e26ea8823cb8"}, + {file = "websockets-11.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7742cd4524622cc7aa71734b51294644492a961243c4fe67874971c4d3045982"}, + {file = "websockets-11.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46dda4bc2030c335abe192b94e98686615f9274f6b56f32f2dd661fb303d9d12"}, + {file = "websockets-11.0.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6b2bfa1d884c254b841b0ff79373b6b80779088df6704f034858e4d705a4802"}, + {file = "websockets-11.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1df2413266bf48430ef2a752c49b93086c6bf192d708e4a9920544c74cd2baa6"}, + {file = "websockets-11.0.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf45d273202b0c1cec0f03a7972c655b93611f2e996669667414557230a87b88"}, + {file = "websockets-11.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a09cce3dacb6ad638fdfa3154d9e54a98efe7c8f68f000e55ca9c716496ca67"}, + {file = "websockets-11.0.2-cp311-cp311-win32.whl", hash = "sha256:2174a75d579d811279855df5824676d851a69f52852edb0e7551e0eeac6f59a4"}, + {file = "websockets-11.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:c78ca3037a954a4209b9f900e0eabbc471fb4ebe96914016281df2c974a93e3e"}, + {file = "websockets-11.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a2100b02d1aaf66dc48ff1b2a72f34f6ebc575a02bc0350cc8e9fbb35940166"}, + {file = "websockets-11.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dca9708eea9f9ed300394d4775beb2667288e998eb6f542cdb6c02027430c599"}, + {file = "websockets-11.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:320ddceefd2364d4afe6576195201a3632a6f2e6d207b0c01333e965b22dbc84"}, + {file = "websockets-11.0.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2a573c8d71b7af937852b61e7ccb37151d719974146b5dc734aad350ef55a02"}, + {file = "websockets-11.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:13bd5bebcd16a4b5e403061b8b9dcc5c77e7a71e3c57e072d8dff23e33f70fba"}, + {file = "websockets-11.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:95c09427c1c57206fe04277bf871b396476d5a8857fa1b99703283ee497c7a5d"}, + {file = "websockets-11.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2eb042734e710d39e9bc58deab23a65bd2750e161436101488f8af92f183c239"}, + {file = "websockets-11.0.2-cp37-cp37m-win32.whl", hash = "sha256:5875f623a10b9ba154cb61967f940ab469039f0b5e61c80dd153a65f024d9fb7"}, + {file = "websockets-11.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:634239bc844131863762865b75211a913c536817c0da27f691400d49d256df1d"}, + {file = "websockets-11.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3178d965ec204773ab67985a09f5696ca6c3869afeed0bb51703ea404a24e975"}, + {file = "websockets-11.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:955fcdb304833df2e172ce2492b7b47b4aab5dcc035a10e093d911a1916f2c87"}, + {file = "websockets-11.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb46d2c7631b2e6f10f7c8bac7854f7c5e5288f024f1c137d4633c79ead1e3c0"}, + {file = "websockets-11.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25aae96c1060e85836552a113495db6d857400288161299d77b7b20f2ac569f2"}, + {file = "websockets-11.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2abeeae63154b7f63d9f764685b2d299e9141171b8b896688bd8baec6b3e2303"}, + {file = "websockets-11.0.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daa1e8ea47507555ed7a34f8b49398d33dff5b8548eae3de1dc0ef0607273a33"}, + {file = "websockets-11.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:954eb789c960fa5daaed3cfe336abc066941a5d456ff6be8f0e03dd89886bb4c"}, + {file = "websockets-11.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3ffe251a31f37e65b9b9aca5d2d67fd091c234e530f13d9dce4a67959d5a3fba"}, + {file = "websockets-11.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:adf6385f677ed2e0b021845b36f55c43f171dab3a9ee0ace94da67302f1bc364"}, + {file = "websockets-11.0.2-cp38-cp38-win32.whl", hash = "sha256:aa7b33c1fb2f7b7b9820f93a5d61ffd47f5a91711bc5fa4583bbe0c0601ec0b2"}, + {file = "websockets-11.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:220d5b93764dd70d7617f1663da64256df7e7ea31fc66bc52c0e3750ee134ae3"}, + {file = "websockets-11.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fb4480556825e4e6bf2eebdbeb130d9474c62705100c90e59f2f56459ddab42"}, + {file = "websockets-11.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec00401846569aaf018700249996143f567d50050c5b7b650148989f956547af"}, + {file = "websockets-11.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87c69f50281126dcdaccd64d951fb57fbce272578d24efc59bce72cf264725d0"}, + {file = "websockets-11.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:232b6ba974f5d09b1b747ac232f3a3d8f86de401d7b565e837cc86988edf37ac"}, + {file = "websockets-11.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:392d409178db1e46d1055e51cc850136d302434e12d412a555e5291ab810f622"}, + {file = "websockets-11.0.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4fe2442091ff71dee0769a10449420fd5d3b606c590f78dd2b97d94b7455640"}, + {file = "websockets-11.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ede13a6998ba2568b21825809d96e69a38dc43184bdeebbde3699c8baa21d015"}, + {file = "websockets-11.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4c54086b2d2aec3c3cb887ad97e9c02c6be9f1d48381c7419a4aa932d31661e4"}, + {file = "websockets-11.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e37a76ccd483a6457580077d43bc3dfe1fd784ecb2151fcb9d1c73f424deaeba"}, + {file = "websockets-11.0.2-cp39-cp39-win32.whl", hash = "sha256:d1881518b488a920434a271a6e8a5c9481a67c4f6352ebbdd249b789c0467ddc"}, + {file = "websockets-11.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:25e265686ea385f22a00cc2b719b880797cd1bb53b46dbde969e554fb458bfde"}, + {file = "websockets-11.0.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ce69f5c742eefd039dce8622e99d811ef2135b69d10f9aa79fbf2fdcc1e56cd7"}, + {file = "websockets-11.0.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b985ba2b9e972cf99ddffc07df1a314b893095f62c75bc7c5354a9c4647c6503"}, + {file = "websockets-11.0.2-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b52def56d2a26e0e9c464f90cadb7e628e04f67b0ff3a76a4d9a18dfc35e3dd"}, + {file = "websockets-11.0.2-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d70a438ef2a22a581d65ad7648e949d4ccd20e3c8ed7a90bbc46df4e60320891"}, + {file = "websockets-11.0.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:752fbf420c71416fb1472fec1b4cb8631c1aa2be7149e0a5ba7e5771d75d2bb9"}, + {file = "websockets-11.0.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:dd906b0cdc417ea7a5f13bb3c6ca3b5fd563338dc596996cb0fdd7872d691c0a"}, + {file = "websockets-11.0.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e79065ff6549dd3c765e7916067e12a9c91df2affea0ac51bcd302aaf7ad207"}, + {file = "websockets-11.0.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46388a050d9e40316e58a3f0838c63caacb72f94129eb621a659a6e49bad27ce"}, + {file = "websockets-11.0.2-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c7de298371d913824f71b30f7685bb07ad13969c79679cca5b1f7f94fec012f"}, + {file = "websockets-11.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6d872c972c87c393e6a49c1afbdc596432df8c06d0ff7cd05aa18e885e7cfb7c"}, + {file = "websockets-11.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b444366b605d2885f0034dd889faf91b4b47668dd125591e2c64bfde611ac7e1"}, + {file = "websockets-11.0.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b967a4849db6b567dec3f7dd5d97b15ce653e3497b8ce0814e470d5e074750"}, + {file = "websockets-11.0.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2acdc82099999e44fa7bd8c886f03c70a22b1d53ae74252f389be30d64fd6004"}, + {file = "websockets-11.0.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:518ed6782d9916c5721ebd61bb7651d244178b74399028302c8617d0620af291"}, + {file = "websockets-11.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:58477b041099bb504e1a5ddd8aa86302ed1d5c6995bdd3db2b3084ef0135d277"}, + {file = "websockets-11.0.2-py3-none-any.whl", hash = "sha256:5004c087d17251938a52cce21b3dbdabeecbbe432ce3f5bbbf15d8692c36eac9"}, + {file = "websockets-11.0.2.tar.gz", hash = "sha256:b1a69701eb98ed83dd099de4a686dc892c413d974fa31602bc00aca7cb988ac9"}, ] -[package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - [[package]] name = "werkzeug" version = "2.2.3" @@ -1510,4 +1573,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "010faf1151a864dfd8f7d72f16be9457675f1359dcb30393ed2ca8db47b3887c" +content-hash = "eee5683fb3dcc73b282ac84c49503087af4b618d1c03d9919fb099e0c2395fb0" diff --git a/pyproject.toml b/pyproject.toml index 32b6df29b2..846c307e13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,9 +26,9 @@ xmltojson = "^2.0.2" pycryptodome = "^3.17" idna = "^3.4" ansible = "^7.3.0" -websocket-client = "^1.5.1" tabulate = "0.8.10" cloudcheck = "^2.0.0.34" +websockets = "^11.0.2" [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" From 8c630c7de4a046318f24440a5134babb11b8c845 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 5 May 2023 17:09:23 -0400 Subject: [PATCH 006/387] working http helpers + interactsh --- bbot/cli.py | 5 +- bbot/core/configurator/args.py | 40 ++++--- bbot/core/helpers/dns.py | 56 +++++----- bbot/core/helpers/helper.py | 10 +- bbot/core/helpers/interactsh.py | 46 ++++---- bbot/core/helpers/misc.py | 7 ++ bbot/core/helpers/web.py | 146 +++++++++++++++++++++++++ bbot/modules/base.py | 2 +- bbot/modules/output/asset_inventory.py | 4 +- bbot/modules/output/websocket.py | 3 +- bbot/scanner/manager.py | 16 +-- bbot/scanner/scanner.py | 5 +- bbot/test/test.conf | 1 + bbot/test/test_step_2/test_dns.py | 121 ++++++++++++++++++++ bbot/test/test_step_2/test_http.py | 74 +++++++++++++ poetry.lock | 134 ++++++++++++++++++++++- pyproject.toml | 1 + 17 files changed, 582 insertions(+), 89 deletions(-) create mode 100644 bbot/test/test_step_2/test_dns.py create mode 100644 bbot/test/test_step_2/test_http.py diff --git a/bbot/cli.py b/bbot/cli.py index 55ee624f3d..c1c637839f 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -32,11 +32,12 @@ from . import config +err = False scan_name = "" async def _main(): - err = False + global err global scan_name ensure_config_files() @@ -336,7 +337,7 @@ def main(): if scan_name: msg = f"You killed {scan_name}" log_to_stderr(msg, level="ERROR") - err = True + os._exit(1) if __name__ == "__main__": diff --git a/bbot/core/configurator/args.py b/bbot/core/configurator/args.py index 4f9bd7202f..0cb2f1519f 100644 --- a/bbot/core/configurator/args.py +++ b/bbot/core/configurator/args.py @@ -104,8 +104,8 @@ def error(self, message): action="store_true", help="Don't consider subdomains of target/whitelist to be in-scope", ) - p.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") - p.add_argument( + modules = p.add_argument_group(title="Modules") + modules.add_argument( "-m", "--modules", nargs="+", @@ -113,9 +113,11 @@ def error(self, message): help=f'Modules to enable. Choices: {",".join(module_choices)}', metavar="MODULE", ) - p.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") - p.add_argument("-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE") - p.add_argument( + modules.add_argument("-l", "--list-modules", action="store_true", help=f"List available modules.") + modules.add_argument( + "-em", "--exclude-modules", nargs="+", default=[], help=f"Exclude these modules.", metavar="MODULE" + ) + modules.add_argument( "-f", "--flags", nargs="+", @@ -123,7 +125,7 @@ def error(self, message): help=f'Enable modules by flag. Choices: {",".join(sorted(flag_choices))}', metavar="FLAG", ) - p.add_argument( + modules.add_argument( "-rf", "--require-flags", nargs="+", @@ -131,7 +133,7 @@ def error(self, message): help=f"Only enable modules with these flags (e.g. -rf passive)", metavar="FLAG", ) - p.add_argument( + modules.add_argument( "-ef", "--exclude-flags", nargs="+", @@ -139,7 +141,7 @@ def error(self, message): help=f"Disable modules with these flags. (e.g. -ef aggressive)", metavar="FLAG", ) - p.add_argument( + modules.add_argument( "-om", "--output-modules", nargs="+", @@ -147,26 +149,28 @@ def error(self, message): help=f'Output module(s). Choices: {",".join(output_module_choices)}', metavar="MODULE", ) - p.add_argument( + modules.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") + scan = p.add_argument_group(title="Scan") + scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") + scan.add_argument( "-o", "--output-dir", metavar="DIR", ) - p.add_argument( + scan.add_argument( "-c", "--config", nargs="*", help="custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'", metavar="CONFIG", ) - p.add_argument("--allow-deadly", action="store_true", help="Enable the use of highly aggressive modules") - p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") - p.add_argument("-d", "--debug", action="store_true", help="Enable debugging") - p.add_argument("-s", "--silent", action="store_true", help="Be quiet") - p.add_argument("--force", action="store_true", help="Run scan even if module setups fail") - p.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") - p.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") - p.add_argument( + scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") + scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") + scan.add_argument("--force", action="store_true", help="Run scan even if module setups fail") + scan.add_argument("-y", "--yes", action="store_true", help="Skip scan confirmation prompt") + scan.add_argument("--dry-run", action="store_true", help=f"Abort before executing scan") + scan.add_argument( "--current-config", action="store_true", help="Show current config in YAML format", diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 1089206a61..56e24262fd 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -1,4 +1,3 @@ -import json import asyncio import logging import ipaddress @@ -84,32 +83,33 @@ async def resolve_raw(self, query, **kwargs): # DNS over TCP is more reliable # But setting this breaks DNS resolution on Ubuntu because systemd-resolve doesn't support TCP # kwargs["tcp"] = True - query = str(query).strip() - if is_ip(query): - kwargs.pop("type", None) - kwargs.pop("rdtype", None) - results, errors = await self._resolve_ip(query, **kwargs) - return [("PTR", results)], [("PTR", e) for e in errors] - else: + with suppress(asyncio.CancelledError): results = [] errors = [] - types = ["A", "AAAA"] - kwargs.pop("rdtype", None) - if "type" in kwargs: - t = kwargs.pop("type") - if isinstance(t, str): - if t.strip().lower() in ("any", "all", "*"): - types = self.all_rdtypes - else: - types = [t.strip().upper()] - elif any([isinstance(t, x) for x in (list, tuple)]): - types = [str(_).strip().upper() for _ in t] - for t in types: - r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) - if r: - results.append((t, r)) - for error in e: - errors.append((t, error)) + query = str(query).strip() + if is_ip(query): + kwargs.pop("type", None) + kwargs.pop("rdtype", None) + results, errors = await self._resolve_ip(query, **kwargs) + return [("PTR", results)], [("PTR", e) for e in errors] + else: + types = ["A", "AAAA"] + kwargs.pop("rdtype", None) + if "type" in kwargs: + t = kwargs.pop("type") + if isinstance(t, str): + if t.strip().lower() in ("any", "all", "*"): + types = self.all_rdtypes + else: + types = [t.strip().upper()] + elif any([isinstance(t, x) for x in (list, tuple)]): + types = [str(_).strip().upper() for _ in t] + for t in types: + r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) + if r: + results.append((t, r)) + for error in e: + errors.append((t, error)) return (results, errors) @@ -277,7 +277,7 @@ async def resolve_event(self, event, minimal=False): types = ("A", "AAAA") if types: - tasks = [self.resolve_raw(event_host, type=t, cache_result=True) for t in types] + tasks = [asyncio.create_task(self.resolve_raw(event_host, type=t, cache_result=True)) for t in types] for task in asyncio.as_completed(tasks): resolved_raw, errors = await task for rdtype, e in errors: @@ -454,7 +454,9 @@ async def is_wildcard(self, query, ips=None, rdtype=None): # then resolve the query for all rdtypes for _rdtype in self.all_rdtypes: # resolve the base query - wildcard_tasks[_rdtype].append(self.resolve_raw(query, type=_rdtype, cache_result=True)) + wildcard_tasks[_rdtype].append( + asyncio.create_task(self.resolve_raw(query, type=_rdtype, cache_result=True)) + ) for _rdtype, tasks in wildcard_tasks.items(): for task in asyncio.as_completed(tasks): diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index db6f1b74a8..f7c345e544 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -5,6 +5,7 @@ from . import misc from .dns import DNSHelper +from .web import WebHelper from .diff import HttpCompare from .wordcloud import WordCloud from .cloud import CloudProviders @@ -45,6 +46,7 @@ def __init__(self, config, scan=None): self._future_lock = Lock() self.dns = DNSHelper(self) + self.web = WebHelper(self) self.depsinstaller = DepsInstaller(self) self.word_cloud = WordCloud(self) self.dummy_modules = {} @@ -114,8 +116,12 @@ def __getattribute__(self, attr): # then try dns return getattr(self.dns, attr) except AttributeError: - # then die - raise AttributeError(f'Helper has no attribute "{attr}"') + try: + # then try web + return getattr(self.web, attr) + except AttributeError: + # then die + raise AttributeError(f'Helper has no attribute "{attr}"') class DummyModule(BaseModule): diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index 42b456c6ee..b79c4cb6a3 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -2,11 +2,10 @@ import json import base64 import random +import asyncio import logging import traceback -from time import sleep from uuid import uuid4 -from threading import Thread from Crypto.Hash import SHA256 from Crypto.PublicKey import RSA @@ -26,9 +25,9 @@ def __init__(self, parent_helper): self.correlation_id = None self.custom_server = self.parent_helper.config.get("interactsh_server", None) self.token = self.parent_helper.config.get("interactsh_token", None) - self._thread = None + self._poll_task = None - def register(self, callback=None): + async def register(self, callback=None): rsa = RSA.generate(1024) self.public_key = rsa.publickey().exportKey() @@ -57,7 +56,9 @@ def register(self, callback=None): "secret-key": self.secret, "correlation-id": self.correlation_id, } - r = self.parent_helper.request(f"https://{server}/register", headers=headers, json=data, method="POST") + r = await self.parent_helper.request_async( + f"https://{server}/register", headers=headers, json=data, method="POST" + ) if r is None: continue try: @@ -78,12 +79,11 @@ def register(self, callback=None): ) if callable(callback): - self._thread = Thread(target=self.poll_loop, args=(callback,), daemon=True) - self._thread.start() + self._poll_task = asyncio.create_task(self.poll_loop(callback)) return self.domain - def deregister(self): + async def deregister(self): if not self.server or not self.correlation_id or not self.secret: raise InteractshError(f"Missing required information to deregister") @@ -93,11 +93,17 @@ def deregister(self): data = {"secret-key": self.secret, "correlation-id": self.correlation_id} - r = self.parent_helper.request(f"https://{self.server}/deregister", headers=headers, json=data, method="POST") + r = await self.parent_helper.request_async( + f"https://{self.server}/deregister", headers=headers, json=data, method="POST" + ) + + if self._poll_task is not None: + self._poll_task.cancel() + if "success" not in getattr(r, "text", ""): raise InteractshError(f"Failed to de-register with interactsh server {self.server}") - def poll(self): + async def poll(self): if not self.server or not self.correlation_id or not self.secret: raise InteractshError(f"Missing required information to poll") @@ -105,38 +111,40 @@ def poll(self): if self.token: headers["Authorization"] = self.token - r = self.parent_helper.request( + r = await self.parent_helper.request_async( f"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}", headers=headers ) + ret = [] data_list = r.json().get("data", None) if data_list: aes_key = r.json()["aes_key"] for data in data_list: decrypted_data = self.decrypt(aes_key, data) - yield decrypted_data + ret.append(decrypted_data) + return ret - def poll_loop(self, callback): - return self.parent_helper.scan.manager.catch(self._poll_loop, callback, _force=True) + async def poll_loop(self, callback): + return await self.parent_helper.scan.manager.catch(self._poll_loop, callback, _force=True) - def _poll_loop(self, callback): + async def _poll_loop(self, callback): while 1: if self.parent_helper.scan.stopping: - sleep(1) + await asyncio.sleep(1) continue data_list = [] try: - data_list = list(self.poll()) + data_list = await self.poll() except InteractshError as e: log.warning(e) log.trace(traceback.format_exc()) if not data_list: - sleep(10) + await asyncio.sleep(10) continue for data in data_list: if data: - callback(data) + self.parent_helper.execute_sync_or_async(callback, data) def decrypt(self, aes_key, data): private_key = RSA.importKey(self.private_key) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index e65766843a..8395d285a0 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1024,3 +1024,10 @@ def cloudcheck(ip): def is_async_function(f): return inspect.iscoroutinefunction(f) + + +async def execute_sync_or_async(callback, *args, **kwargs): + if is_async_function(callback): + return await callback(*args, **kwargs) + else: + return callback(*args, **kwargs) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index d59acb0266..60d7b88cdd 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -1,3 +1,4 @@ +import httpx import logging import requests from time import sleep @@ -169,6 +170,151 @@ def request(self, *args, **kwargs): raise e +class BBOTAsyncClient(httpx.AsyncClient): + def __init__(self, *args, **kwargs): + self._bbot_scan = kwargs.pop("_bbot_scan") + + # timeout + http_timeout = self._bbot_scan.config.get("http_timeout", 20) + if not "timeout" in kwargs: + kwargs["timeout"] = http_timeout + + # headers + headers = kwargs.get("headers", None) + if headers is None: + headers = {} + user_agent = self._bbot_scan.config.get("user_agent", "BBOT") + if "User-Agent" not in headers: + headers["User-Agent"] = user_agent + kwargs["headers"] = headers + + super().__init__(*args, **kwargs) + + def build_request(self, *args, **kwargs): + request = super().build_request(*args, **kwargs) + # add custom headers if the URL is in-scope + if self._bbot_scan.in_scope(str(request.url)): + for hk, hv in self._bbot_scan.config.get("http_headers", {}).items(): + # don't clobber headers + if hk not in request.headers: + request.headers[hk] = hv + return request + + +class WebHelper: + """ + For making HTTP requests + """ + + client_options = ("auth", "params", "headers", "cookies", "timeout", "follow_redirects", "max_redirects") + + def __init__(self, parent_helper): + self.parent_helper = parent_helper + + def AsyncClient(self, *args, **kwargs): + kwargs["_bbot_scan"] = self.parent_helper.scan + retries = kwargs.pop("retries", self.parent_helper.config.get("http_retries", 1)) + kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries) + return BBOTAsyncClient(*args, **kwargs) + + async def request_async(self, *args, **kwargs): + raise_error = kwargs.pop("raise_error", False) + + # in case of URL only, assume GET request + if len(args) == 1: + kwargs["url"] = args[0] + args = [] + + if not args and "method" not in kwargs: + kwargs["method"] = "GET" + + http_debug = self.parent_helper.config.get("http_debug", False) + + client_kwargs = {} + for k in list(kwargs): + if k in self.client_options: + v = kwargs.pop(k) + client_kwargs[k] = v + async with self.AsyncClient(**client_kwargs) as client: + try: + if http_debug: + logstr = f"Web request: {str(args)}, {str(kwargs)}" + log.debug(logstr) + response = await client.request(*args, **kwargs) + if http_debug: + log.debug( + f"Web response: {response} (Length: {len(response.content)}) headers: {response.headers}" + ) + return response + except httpx.RequestError as e: + log.debug(f"Error with request: {e}") + if raise_error: + raise + + async def download_async(self, url, **kwargs): + """ + Downloads file, returns full path of filename + If download failed, returns None + + Caching supported via "cache_hrs" + """ + success = False + filename = self.parent_helper.cache_filename(url) + cache_hrs = float(kwargs.pop("cache_hrs", -1)) + log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}") + if cache_hrs > 0 and self.parent_helper.is_cached(url): + log.debug(f"{url} is cached") + success = True + else: + # kwargs["raise_error"] = True + # kwargs["stream"] = True + if not "method" in kwargs: + kwargs["method"] = "GET" + try: + async with self.AsyncClient().stream(url=url, **kwargs) as response: + status_code = getattr(response, "status_code", 0) + log.debug(f"Download result: HTTP {status_code}") + if status_code != 0: + response.raise_for_status() + with open(filename, "wb") as f: + async for chunk in response.aiter_bytes(chunk_size=8192): + f.write(chunk) + success = True + except httpx.HTTPError as e: + log.warning(f"Failed to download {url}: {e}") + return + + if success: + return filename.resolve() + + async def wordlist_async(self, path, lines=None, **kwargs): + if not path: + raise WordlistError(f"Invalid wordlist: {path}") + if not "cache_hrs" in kwargs: + kwargs["cache_hrs"] = 720 + if self.parent_helper.is_url(path): + filename = await self.download_async(str(path), **kwargs) + if filename is None: + raise WordlistError(f"Unable to retrieve wordlist from {path}") + else: + filename = Path(path).resolve() + if not filename.is_file(): + raise WordlistError(f"Unable to find wordlist at {path}") + + if lines is None: + return filename + else: + lines = int(lines) + with open(filename) as f: + read_lines = f.readlines() + cache_key = f"{filename}:{lines}" + truncated_filename = self.parent_helper.cache_filename(cache_key) + with open(truncated_filename, "w") as f: + for line in read_lines[:lines]: + f.write(line) + return truncated_filename + + def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs): page = 1 offset = 0 diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 888a56f9a7..e4c23e9d90 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1,7 +1,7 @@ import queue import asyncio import logging -import threading +import traceback from sys import exc_info from contextlib import suppress diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index f904728556..5ffb3d86ae 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -55,14 +55,14 @@ def filter_event(self, event): return False, "event is unresolved" return True, "" - def handle_event(self, event): + async def handle_event(self, event): if self.filter_event(event)[0]: hostkey = _make_hostkey(event.host, event.resolved_hosts) if hostkey not in self.assets: self.assets[hostkey] = Asset(event.host) self.assets[hostkey].absorb_event(event) - def report(self): + async def report(self): stats = dict() totals = dict() diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index 7e302d0b66..9c78e42d10 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -1,7 +1,6 @@ import json -import threading +import asyncio import websockets -from time import sleep from bbot.modules.output.base import BaseOutputModule diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 5a18cdac2f..e2bf595064 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -2,9 +2,7 @@ import asyncio import logging import traceback -from time import sleep from contextlib import suppress -from datetime import datetime, timedelta from ..core.helpers.queueing import EventQueue from ..core.helpers.async_helpers import TaskCounter @@ -30,8 +28,6 @@ def __init__(self, scan): self._task_counter = TaskCounter() self._new_activity = True - self.last_log_time = datetime.now() - async def init_events(self): """ seed scanner with target events @@ -341,10 +337,7 @@ async def catch(self, callback, *args, **kwargs): break try: if not self.scan.stopping or force: - if self.scan.helpers.is_async_function(callback): - ret = await callback(*args, **kwargs) - else: - ret = callback(*args, **kwargs) + ret = await self.scan.helpers.execute_sync_or_async(callback, *args, **kwargs) except ScanCancelledError as e: log.debug(f"ScanCancelledError in {fn.__qualname__}(): {e}") except BrokenPipeError as e: @@ -355,12 +348,11 @@ async def catch(self, callback, *args, **kwargs): except KeyboardInterrupt: log.debug(f"Interrupted") self.scan.stop() + except asyncio.CancelledError as e: + log.debug(f"{e}") if callable(on_finish_callback): try: - if self.scan.helpers.is_async_function(on_finish_callback): - await on_finish_callback() - else: - on_finish_callback() + await self.scan.helpers.execute_sync_or_async(on_finish_callback) except Exception as e: log.error( f"Error in on_finish_callback {on_finish_callback.__qualname__}() after {fn.__qualname__}(): {e}" diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 11df8d770d..465d2f7a4a 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1,13 +1,13 @@ +import queue import asyncio import logging import traceback from sys import exc_info from pathlib import Path -import concurrent.futures from datetime import datetime from omegaconf import OmegaConf from contextlib import suppress -from collections import OrderedDict, deque +from collections import OrderedDict from bbot import config as bbot_config @@ -321,7 +321,6 @@ async def setup_modules(self, remove_failed=True): self.verbose(f"Setting up modules") hard_failed = [] soft_failed = [] - setup_futures = dict() for task in asyncio.as_completed([m._setup() for m in self.modules.values()]): module_name, status, msg = await task diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 597d313fc4..5e534f5067 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -30,6 +30,7 @@ scope_report_distance: 0 scope_dns_search_distance: 1 plumbus: asdf dns_debug: true +user_agent: "BBOT Test User-Agent" http_debug: false keep_scans: 1 agent_url: test diff --git a/bbot/test/test_step_2/test_dns.py b/bbot/test/test_step_2/test_dns.py new file mode 100644 index 0000000000..ab1bcecc18 --- /dev/null +++ b/bbot/test/test_step_2/test_dns.py @@ -0,0 +1,121 @@ +from ..bbot_fixtures import * + + +@pytest.mark.asyncio +async def test_dns(bbot_scanner, bbot_config): + scan = bbot_scanner("8.8.8.8") + helpers = scan.helpers + + # lowest level functions + a_responses = await helpers._resolve_hostname("dns.google") + aaaa_responses = await helpers._resolve_hostname("dns.google", rdtype="AAAA") + ip_responses = await helpers._resolve_ip("8.8.8.8") + assert a_responses[0].response.answer[0][0].address in ("8.8.8.8", "8.8.4.4") + assert aaaa_responses[0].response.answer[0][0].address in ("2001:4860:4860::8888", "2001:4860:4860::8844") + assert ip_responses[0].response.answer[0][0].target.to_text() in ("dns.google.",) + + # mid level functions + _responses, errors = await helpers.resolve_raw("dns.google") + responses = [] + for rdtype, response in _responses: + for answers in response: + responses += list(helpers.extract_targets(answers)) + assert ("A", "8.8.8.8") in responses + _responses, errors = await helpers.resolve_raw("dns.google", rdtype="AAAA") + responses = [] + for rdtype, response in _responses: + for answers in response: + responses += list(helpers.extract_targets(answers)) + assert ("AAAA", "2001:4860:4860::8888") in responses + _responses, errors = await helpers.resolve_raw("8.8.8.8") + responses = [] + for rdtype, response in _responses: + for answers in response: + responses += list(helpers.extract_targets(answers)) + assert ("PTR", "dns.google") in responses + + # high level functions + assert "8.8.8.8" in await helpers.resolve("dns.google") + assert "2001:4860:4860::8888" in await helpers.resolve("dns.google", type="AAAA") + assert "dns.google" in await helpers.resolve("8.8.8.8") + for rdtype in ("NS", "SOA", "MX", "TXT"): + assert len(await helpers.resolve("google.com", type=rdtype)) > 0 + + # batch resolution + batch_results = [r async for r in helpers.resolve_batch(["8.8.8.8", "dns.google"])] + assert len(batch_results) == 2 + batch_results = dict(batch_results) + assert any([x in batch_results["dns.google"] for x in ("8.8.8.8", "8.8.4.4")]) + assert "dns.google" in batch_results["8.8.8.8"] + + # "any" type + resolved = await helpers.resolve("google.com", type="any") + assert any([helpers.is_subdomain(h) for h in resolved]) + + # dns cache + assert hash(f"8.8.8.8:PTR") not in helpers.dns._dns_cache + assert hash(f"dns.google:A") not in helpers.dns._dns_cache + assert hash(f"dns.google:AAAA") not in helpers.dns._dns_cache + await helpers.resolve("8.8.8.8", cache_result=True) + assert hash(f"8.8.8.8:PTR") in helpers.dns._dns_cache + await helpers.resolve("dns.google", cache_result=True) + assert hash(f"dns.google:A") in helpers.dns._dns_cache + assert hash(f"dns.google:AAAA") in helpers.dns._dns_cache + + # wildcards + wildcard_domains = await helpers.is_wildcard_domain("asdf.github.io") + assert "github.io" in wildcard_domains + assert "A" in wildcard_domains["github.io"] + assert "SRV" not in wildcard_domains["github.io"] + assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"]) + + wildcard_rdtypes = await helpers.is_wildcard("blacklanternsecurity.github.io") + assert "A" in wildcard_rdtypes + assert "SRV" not in wildcard_rdtypes + assert wildcard_rdtypes["A"] == (True, "github.io") + assert hash("github.io") in helpers.dns._wildcard_cache + assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 + helpers.dns._wildcard_cache.clear() + + wildcard_rdtypes = await helpers.is_wildcard("asdf.asdf.asdf.github.io") + assert "A" in wildcard_rdtypes + assert "SRV" not in wildcard_rdtypes + assert wildcard_rdtypes["A"] == (True, "github.io") + assert hash("github.io") in helpers.dns._wildcard_cache + assert len(helpers.dns._wildcard_cache[hash("github.io")]) > 0 + wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", dummy=True) + wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", dummy=True) + wildcard_event3 = scan.make_event("github.io", "DNS_NAME", dummy=True) + + # event resolution + event_tags1, event_whitelisted1, event_blacklisted1, children1 = await scan.helpers.resolve_event(wildcard_event1) + event_tags2, event_whitelisted2, event_blacklisted2, children2 = await scan.helpers.resolve_event(wildcard_event2) + event_tags3, event_whitelisted3, event_blacklisted3, children3 = await scan.helpers.resolve_event(wildcard_event3) + await helpers.handle_wildcard_event(wildcard_event1, children1) + await helpers.handle_wildcard_event(wildcard_event2, children2) + await helpers.handle_wildcard_event(wildcard_event3, children3) + assert "wildcard" in wildcard_event1.tags + assert "a-wildcard" in wildcard_event1.tags + assert "srv-wildcard" not in wildcard_event1.tags + assert "wildcard" in wildcard_event2.tags + assert "a-wildcard" in wildcard_event2.tags + assert "srv-wildcard" not in wildcard_event2.tags + assert wildcard_event1.data == "_wildcard.github.io" + assert wildcard_event2.data == "_wildcard.github.io" + assert wildcard_event1.tags == wildcard_event2.tags + assert "wildcard-domain" in wildcard_event3.tags + assert "a-wildcard-domain" in wildcard_event3.tags + assert "srv-wildcard-domain" not in wildcard_event3.tags + + # Ensure events with hosts have resolved_hosts attribute populated + resolved_hosts_event1 = scan.make_event("dns.google", "DNS_NAME", dummy=True) + resolved_hosts_event2 = scan.make_event("http://dns.google/", "URL_UNVERIFIED", dummy=True) + event_tags1, event_whitelisted1, event_blacklisted1, children1 = await scan.helpers.resolve_event( + resolved_hosts_event1 + ) + event_tags2, event_whitelisted2, event_blacklisted2, children2 = await scan.helpers.resolve_event( + resolved_hosts_event2 + ) + assert "8.8.8.8" in [str(x) for x in children1["A"]] + assert "8.8.8.8" in [str(x) for x in children2["A"]] + assert set(children1.keys()) == set(children2.keys()) diff --git a/bbot/test/test_step_2/test_http.py b/bbot/test/test_step_2/test_http.py new file mode 100644 index 0000000000..47b6c584dc --- /dev/null +++ b/bbot/test/test_step_2/test_http.py @@ -0,0 +1,74 @@ +from ..bbot_fixtures import * + + +@pytest.mark.asyncio +async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): + scan1 = bbot_scanner("8.8.8.8", config=bbot_config) + scan2 = bbot_scanner("127.0.0.1", config=bbot_config) + + user_agent = bbot_config.get("user_agent", "") + headers = {"User-Agent": user_agent} + custom_headers = bbot_config.get("http_headers", {}) + headers.update(custom_headers) + assert headers["test"] == "header" + + url = bbot_httpserver.url_for("/test_http_helpers") + # test user agent + custom headers + bbot_httpserver.expect_request(uri="/test_http_helpers", headers=headers).respond_with_data( + "test_http_helpers_yep" + ) + response = await scan1.helpers.request_async(url) + # should fail because URL is not in-scope + assert response.status_code == 500 + response = await scan2.helpers.request_async(url) + # should suceed because URL is in-scope + assert response.status_code == 200 + assert response.text == "test_http_helpers_yep" + + # download file + path = "/test_http_helpers_download" + url = bbot_httpserver.url_for(path) + download_content = "test_http_helpers_download_yep" + bbot_httpserver.expect_request(uri=path).respond_with_data(download_content) + filename = await scan1.helpers.download_async(url) + assert Path(str(filename)).is_file() + assert scan1.helpers.is_cached(url) + with open(filename) as f: + assert f.read() == download_content + # 404 + path = "/test_http_helpers_download_404" + url = bbot_httpserver.url_for(path) + download_content = "404" + bbot_httpserver.expect_request(uri=path).respond_with_data(download_content, status=404) + filename = await scan1.helpers.download_async(url) + assert filename is None + assert not scan1.helpers.is_cached(url) + + # wordlist + path = "/test_http_helpers_wordlist" + url = bbot_httpserver.url_for(path) + download_content = "a\ncool\nword\nlist" + bbot_httpserver.expect_request(uri=path).respond_with_data(download_content) + filename = await scan1.helpers.wordlist_async(url) + assert Path(str(filename)).is_file() + assert scan1.helpers.is_cached(url) + with open(filename) as f: + assert f.read().splitlines() == ["a", "cool", "word", "list"] + + +@pytest.mark.asyncio +async def test_http_interactsh(bbot_scanner, bbot_config, bbot_httpserver): + from bbot.core.helpers.interactsh import server_list + + scan1 = bbot_scanner("8.8.8.8", config=bbot_config) + + interactsh_client = scan1.helpers.interactsh() + + async def async_callback(data): + log.debug(f"interactsh poll: {data}") + + interactsh_domain = await interactsh_client.register(callback=async_callback) + assert any(interactsh_domain.endswith(f"{s}") for s in server_list) + data_list = await interactsh_client.poll() + assert isinstance(data_list, list) + assert await interactsh_client.deregister() is None diff --git a/poetry.lock b/poetry.lock index a032ccadeb..f0fdf18237 100644 --- a/poetry.lock +++ b/poetry.lock @@ -64,6 +64,27 @@ files = [ {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, ] +[[package]] +name = "anyio" +version = "3.6.2" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16,<0.22)"] + [[package]] name = "appdirs" version = "1.4.4" @@ -602,6 +623,105 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.10.0,<2.11.0" pyflakes = ">=3.0.0,<3.1.0" +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "h2" +version = "4.1.0" +description = "HTTP/2 State-Machine based protocol implementation" +category = "main" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "h2-4.1.0-py3-none-any.whl", hash = "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d"}, + {file = "h2-4.1.0.tar.gz", hash = "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"}, +] + +[package.dependencies] +hpack = ">=4.0,<5" +hyperframe = ">=6.0,<7" + +[[package]] +name = "hpack" +version = "4.0.0" +description = "Pure-Python HPACK header compression" +category = "main" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "hpack-4.0.0-py3-none-any.whl", hash = "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c"}, + {file = "hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"}, +] + +[[package]] +name = "httpcore" +version = "0.17.0" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpcore-0.17.0-py3-none-any.whl", hash = "sha256:0fdfea45e94f0c9fd96eab9286077f9ff788dd186635ae61b312693e4d943599"}, + {file = "httpcore-0.17.0.tar.gz", hash = "sha256:cc045a3241afbf60ce056202301b4d8b6af08845e3294055eb26b09913ef903c"}, +] + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] + +[[package]] +name = "httpx" +version = "0.24.0" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpx-0.24.0-py3-none-any.whl", hash = "sha256:447556b50c1921c351ea54b4fe79d91b724ed2b027462ab9a329465d147d5a4e"}, + {file = "httpx-0.24.0.tar.gz", hash = "sha256:507d676fc3e26110d41df7d35ebd8b3b8585052450f4097401c9be59d928c63e"}, +] + +[package.dependencies] +certifi = "*" +h2 = {version = ">=3,<5", optional = true, markers = "extra == \"http2\""} +httpcore = ">=0.15.0,<0.18.0" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] + +[[package]] +name = "hyperframe" +version = "6.0.1" +description = "HTTP/2 framing layer for Python" +category = "main" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "hyperframe-6.0.1-py3-none-any.whl", hash = "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15"}, + {file = "hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"}, +] + [[package]] name = "idna" version = "3.4" @@ -1333,6 +1453,18 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + [[package]] name = "tabulate" version = "0.8.10" @@ -1573,4 +1705,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "eee5683fb3dcc73b282ac84c49503087af4b618d1c03d9919fb099e0c2395fb0" +content-hash = "7d520e30d4c148bb627da6aea282645e5e3452faeb11798bef27ae81d1941f88" diff --git a/pyproject.toml b/pyproject.toml index 846c307e13..cebea306f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ ansible = "^7.3.0" tabulate = "0.8.10" cloudcheck = "^2.0.0.34" websockets = "^11.0.2" +httpx = {extras = ["http2"], version = "^0.24.0"} [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" From eaffce0d13bd6dc251de6a535001f0e8d36f8dd7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 May 2023 16:33:50 -0400 Subject: [PATCH 007/387] subprocess, massdns working --- bbot/core/helpers/command.py | 245 ++++++----------- bbot/core/helpers/files.py | 102 ++++++++ bbot/core/helpers/helper.py | 6 +- bbot/core/helpers/interactsh.py | 6 +- bbot/core/helpers/names_generator.py | 3 + bbot/core/helpers/web.py | 361 +++++++------------------- bbot/modules/base.py | 8 +- bbot/modules/crobat.py | 16 +- bbot/modules/internal/speculate.py | 2 +- bbot/modules/massdns.py | 48 ++-- bbot/scanner/scanner.py | 4 +- bbot/test/test_step_2/test_command.py | 63 +++++ bbot/test/test_step_2/test_files.py | 22 ++ bbot/test/test_step_2/test_helpers.py | 93 ------- bbot/test/test_step_2/test_http.py | 48 +++- poetry.lock | 12 +- pyproject.toml | 1 - 17 files changed, 456 insertions(+), 584 deletions(-) create mode 100644 bbot/core/helpers/files.py create mode 100644 bbot/test/test_step_2/test_command.py create mode 100644 bbot/test/test_step_2/test_files.py diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 03c0d2f04d..65b376bf66 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -1,94 +1,104 @@ -import io import os +import asyncio import logging -import threading import traceback -import subprocess -from contextlib import suppress +from subprocess import CompletedProcess -from .misc import smart_decode, rm_at_exit +from .misc import smart_decode, smart_encode log = logging.getLogger("bbot.core.helpers.command") -def run_live(self, command, *args, **kwargs): +async def run(self, *command, **kwargs): """ - Get live output, line by line, as a process executes - You can also pass input= and pipe data into the process' stdin - - This lets you chain processes like so: + Simple helper for running a command, and getting its output as a string + process = await run(["ls", "/tmp"]) + process.stdout --> "file1.txt\nfile2.txt" + """ + proc, _input = await self._spawn_proc(*command, **kwargs) + if proc is not None: + if _input is not None: + _input = smart_encode(_input) + stdout, stderr = await proc.communicate(_input) + + # surface stderr + stderr = smart_decode(stderr) + stdout = smart_decode(stdout) + if stderr and proc.returncode != 0: + command_str = " ".join(command) + log.warning(f"Stderr for {command_str}:\n\t{stderr}") - ls_process = run_live(["ls", "/etc"]) - grep_process = run_live(["grep", "conf"], input=ls_process) - for line in grep_process: - log.success(line) + return CompletedProcess(command, proc.returncode, stdout, stderr) - - The above is roughly equivalent to: - ls /etc | grep conf - NOTE: STDERR is logged after the process exits, if its exit code is non-zero - If you want to see it immediately, pass stderr=None +async def run_live(self, *command, **kwargs): """ - command, kwargs = self._prepare_command_kwargs(command, kwargs) - _input = kwargs.pop("input", "") - input_msg = "" - if _input: - kwargs["stdin"] = subprocess.PIPE - input_msg = " (with stdin)" - log.hugeverbose(f"run_live{input_msg}: {' '.join(command)}") - try: - with catch(subprocess.Popen, command, *args, **kwargs) as process: - if _input: - if type(_input) in (str, bytes): - _input = (_input,) - self.feed_pipe(process.stdin, _input, text=False) - for line in io.TextIOWrapper(process.stdout, encoding="utf-8", errors="ignore", line_buffering=True): - yield line - - # surface stderr - process.wait() - if process.stderr and process.returncode != 0: - stderr = smart_decode(process.stderr.read()) - if stderr: - command_str = " ".join(command) - log.warning(f"Stderr for {command_str}:\n\t{stderr}") - except AttributeError as e: - if not str(e) == "__enter__": - raise - - -def run(self, command, *args, **kwargs): + Simple helper for running a command and iterating through its output line by line in realtime + async for line in run_live(["ls", "/tmp"]): + log.info(line) """ - Simple helper for running a command, and getting its output as a string - process = run(["ls", "/tmp"]) - process.stdout --> "file1.txt\nfile2.txt" + proc, _input = await self._spawn_proc(*command, **kwargs) + if proc is not None: + input_task = None + if _input is not None: + input_task = asyncio.create_task(_write_stdin(proc, _input)) + + while 1: + line = await proc.stdout.readline() + if not line: + break + yield smart_decode(line).rstrip("\r\n") + + if input_task is not None: + await input_task + await proc.wait() + + # surface stderr + # if proc.stderr and proc.returncode != 0: + # command_str = " ".join(command) + # log.warning(f"Stderr for {command_str}:\n\t{stderr}") + + +async def _spawn_proc(self, *command, **kwargs): + command, kwargs = self._prepare_command_kwargs_async(command, kwargs) + _input = kwargs.pop("input", None) + if _input is not None: + if kwargs.get("stdin") is not None: + raise ValueError("stdin and input arguments may not both be used.") + kwargs["stdin"] = asyncio.subprocess.PIPE - NOTE: STDERR is captured (not displayed) by default. - If you want to see it, self.debug(process.stderr) or pass stderr=None - """ - command, kwargs = self._prepare_command_kwargs(command, kwargs) - if not "text" in kwargs: - kwargs["text"] = True log.hugeverbose(f"run: {' '.join(command)}") - result = catch(subprocess.run, command, *args, **kwargs) + try: + proc = await asyncio.create_subprocess_exec(*command, **kwargs) + return proc, _input + except FileNotFoundError as e: + log.warning(f"{e} - missing executable?") + log.trace(traceback.format_exc()) + return None, None - # surface stderr - if result.stderr and result.returncode != 0: - stderr = smart_decode(result.stderr) - if stderr: - command_str = " ".join(command) - log.warning(f"Stderr for {command_str}:\n\t{stderr}") - return result +async def _write_stdin(proc, _input): + if _input is not None: + if isinstance(_input, str): + proc.stdin.write(smart_encode(_input)) + else: + async for chunk in _input: + proc.stdin.write(smart_encode(chunk) + b"\n") + await proc.stdin.drain() + proc.stdin.close() -def _prepare_command_kwargs(self, command, kwargs): +def _prepare_command_kwargs_async(self, command, kwargs): if not "stdout" in kwargs: - kwargs["stdout"] = subprocess.PIPE + kwargs["stdout"] = asyncio.subprocess.PIPE if not "stderr" in kwargs: - kwargs["stderr"] = subprocess.PIPE + kwargs["stderr"] = asyncio.subprocess.PIPE sudo = kwargs.pop("sudo", False) + if len(command) == 1 and isinstance(command[0], (list, tuple)): + command = command[0] command = [str(s) for s in command] + env = kwargs.get("env", os.environ) if sudo and os.geteuid() != 0: self.depsinstaller.ensure_root() @@ -100,106 +110,3 @@ def _prepare_command_kwargs(self, command, kwargs): LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") command = ["sudo", "-E", "-A", f"LD_LIBRARY_PATH={LD_LIBRARY_PATH}", f"PATH={PATH}"] + command return command, kwargs - - -def catch(callback, *args, **kwargs): - try: - return callback(*args, **kwargs) - except FileNotFoundError as e: - log.warning(f"{e} - missing executable?") - log.trace(traceback.format_exc()) - except BrokenPipeError as e: - log.warning(f"Error in subprocess: {e}") - log.trace(traceback.format_exc()) - - -def tempfile(self, content, pipe=True): - """ - tempfile(["temp", "file", "content"]) --> Path("/home/user/.bbot/temp/pgxml13bov87oqrvjz7a") - - if "pipe" is True (the default), a named pipe is used instead of - a true file, which allows python data to be piped directly into the - process without taking up disk space - """ - filename = self.temp_filename() - rm_at_exit(filename) - try: - if type(content) not in (set, list, tuple): - content = (content,) - if pipe: - os.mkfifo(filename) - self.feed_pipe(filename, content, text=True) - else: - with open(filename, "w", errors="ignore") as f: - for c in content: - line = f"{self.smart_decode(c)}\n" - f.write(line) - except Exception as e: - log.error(f"Error creating temp file: {e}") - log.trace(traceback.format_exc()) - - return filename - - -def _feed_pipe(self, pipe, content, text=True): - try: - if text: - decode_fn = self.smart_decode - newline = "\n" - else: - decode_fn = self.smart_encode - newline = b"\n" - try: - if hasattr(pipe, "write"): - try: - for c in content: - pipe.write(decode_fn(c) + newline) - finally: - with suppress(Exception): - pipe.close() - else: - with open(pipe, "w") as p: - for c in content: - p.write(decode_fn(c) + newline) - except BrokenPipeError: - log.debug(f"Broken pipe in _feed_pipe()") - except ValueError: - log.debug(f"Error _feed_pipe(): {traceback.format_exc()}") - except KeyboardInterrupt: - self.scan.stop() - except Exception as e: - log.error(f"Error in _feed_pipe(): {e}") - log.trace(traceback.format_exc()) - - -def feed_pipe(self, pipe, content, text=True): - t = threading.Thread(target=self._feed_pipe, args=(pipe, content), kwargs={"text": text}, daemon=True) - t.start() - - -def tempfile_tail(self, callback): - """ - Create a named pipe and execute a callback on each line - """ - filename = self.temp_filename() - rm_at_exit(filename) - try: - os.mkfifo(filename) - t = threading.Thread(target=tail, args=(filename, callback), daemon=True) - t.start() - except Exception as e: - log.error(f"Error setting up tail for file {filename}: {e}") - log.trace(traceback.format_exc()) - return - return filename - - -def tail(filename, callback): - try: - with open(filename, errors="ignore") as f: - for line in f: - line = line.rstrip("\r\n") - callback(line) - except Exception as e: - log.error(f"Error tailing file {filename}: {e}") - log.trace(traceback.format_exc()) diff --git a/bbot/core/helpers/files.py b/bbot/core/helpers/files.py new file mode 100644 index 0000000000..27ed71948d --- /dev/null +++ b/bbot/core/helpers/files.py @@ -0,0 +1,102 @@ +import os +import logging +import threading +import traceback +from contextlib import suppress + +from .misc import rm_at_exit + + +log = logging.getLogger("bbot.core.helpers.files") + + +def tempfile(self, content, pipe=True): + """ + tempfile(["temp", "file", "content"]) --> Path("/home/user/.bbot/temp/pgxml13bov87oqrvjz7a") + + if "pipe" is True (the default), a named pipe is used instead of + a true file, which allows python data to be piped directly into the + process without taking up disk space + """ + filename = self.temp_filename() + rm_at_exit(filename) + try: + if type(content) not in (set, list, tuple): + content = (content,) + if pipe: + os.mkfifo(filename) + self.feed_pipe(filename, content, text=True) + else: + with open(filename, "w", errors="ignore") as f: + for c in content: + line = f"{self.smart_decode(c)}\n" + f.write(line) + except Exception as e: + log.error(f"Error creating temp file: {e}") + log.trace(traceback.format_exc()) + + return filename + + +def _feed_pipe(self, pipe, content, text=True): + try: + if text: + decode_fn = self.smart_decode + newline = "\n" + else: + decode_fn = self.smart_encode + newline = b"\n" + try: + if hasattr(pipe, "write"): + try: + for c in content: + pipe.write(decode_fn(c) + newline) + finally: + with suppress(Exception): + pipe.close() + else: + with open(pipe, "w") as p: + for c in content: + p.write(decode_fn(c) + newline) + except BrokenPipeError: + log.debug(f"Broken pipe in _feed_pipe()") + except ValueError: + log.debug(f"Error _feed_pipe(): {traceback.format_exc()}") + except KeyboardInterrupt: + self.scan.stop() + except Exception as e: + log.error(f"Error in _feed_pipe(): {e}") + log.trace(traceback.format_exc()) + + +def feed_pipe(self, pipe, content, text=True): + t = threading.Thread(target=self._feed_pipe, args=(pipe, content), kwargs={"text": text}, daemon=True) + t.start() + + +def tempfile_tail(self, callback): + """ + Create a named pipe and execute a callback on each line + """ + filename = self.temp_filename() + rm_at_exit(filename) + try: + os.mkfifo(filename) + t = threading.Thread(target=tail, args=(filename, callback), daemon=True) + t.start() + except Exception as e: + log.error(f"Error setting up tail for file {filename}: {e}") + log.trace(traceback.format_exc()) + return + return filename + + +def tail(filename, callback): + try: + with open(filename, errors="ignore") as f: + for line in f: + line = line.rstrip("\r\n") + callback(line) + except Exception as e: + log.error(f"Error tailing file {filename}: {e}") + log.trace(traceback.format_exc()) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index f7c345e544..cff12d7848 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -19,12 +19,12 @@ class ConfigAwareHelper: - from .web import wordlist, request, download, api_page_iter, curl - from .cache import cache_get, cache_put, cache_filename, is_cached, CacheDict - from .command import run, run_live, _prepare_command_kwargs, tempfile, feed_pipe, _feed_pipe, tempfile_tail from . import ntlm from . import regexes from . import validators + from .files import tempfile, feed_pipe, _feed_pipe, tempfile_tail + from .cache import cache_get, cache_put, cache_filename, is_cached, CacheDict + from .command import run, run_live, _spawn_proc, _prepare_command_kwargs_async def __init__(self, config, scan=None): self.config = config diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index b79c4cb6a3..f3303960f4 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -56,7 +56,7 @@ async def register(self, callback=None): "secret-key": self.secret, "correlation-id": self.correlation_id, } - r = await self.parent_helper.request_async( + r = await self.parent_helper.request( f"https://{server}/register", headers=headers, json=data, method="POST" ) if r is None: @@ -93,7 +93,7 @@ async def deregister(self): data = {"secret-key": self.secret, "correlation-id": self.correlation_id} - r = await self.parent_helper.request_async( + r = await self.parent_helper.request( f"https://{self.server}/deregister", headers=headers, json=data, method="POST" ) @@ -111,7 +111,7 @@ async def poll(self): if self.token: headers["Authorization"] = self.token - r = await self.parent_helper.request_async( + r = await self.parent_helper.request( f"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}", headers=headers ) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 50521de030..1dae35c4f6 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -44,6 +44,7 @@ "decrypted", "deep", "delicious", + "demented", "demonic", "depraved", "depressed", @@ -77,6 +78,7 @@ "ferocious", "fiendish", "fierce", + "flamboyant", "fleecy", "flirtatious", "flustered", @@ -127,6 +129,7 @@ "liquid", "loveable", "lovely", + "lucid", "malevolent", "malfunctioning", "malicious", diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 60d7b88cdd..3d44e298cc 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -1,175 +1,13 @@ import httpx import logging -import requests -from time import sleep +import traceback from pathlib import Path -from requests_cache import CachedSession -from requests.adapters import HTTPAdapter -from requests_cache.backends import SQLiteCache -from requests.exceptions import RequestException from bbot.core.errors import WordlistError, CurlError log = logging.getLogger("bbot.core.helpers.web") -def wordlist(self, path, lines=None, **kwargs): - if not path: - raise WordlistError(f"Invalid wordlist: {path}") - if not "cache_hrs" in kwargs: - kwargs["cache_hrs"] = 720 - if self.is_url(path): - filename = self.download(str(path), **kwargs) - if filename is None: - raise WordlistError(f"Unable to retrieve wordlist from {path}") - else: - filename = Path(path).resolve() - if not filename.is_file(): - raise WordlistError(f"Unable to find wordlist at {path}") - - if lines is None: - return filename - else: - lines = int(lines) - with open(filename) as f: - read_lines = f.readlines() - cache_key = f"{filename}:{lines}" - truncated_filename = self.cache_filename(cache_key) - with open(truncated_filename, "w") as f: - for line in read_lines[:lines]: - f.write(line) - return truncated_filename - - -def download(self, url, **kwargs): - """ - Downloads file, returns full path of filename - If download failed, returns None - - Caching supported via "cache_hrs" - """ - success = False - filename = self.cache_filename(url) - cache_hrs = float(kwargs.pop("cache_hrs", -1)) - log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}") - if cache_hrs > 0 and self.is_cached(url): - log.debug(f"{url} is cached") - success = True - else: - method = kwargs.get("method", "GET") - try: - with self.request(method=method, url=url, stream=True, raise_error=True, **kwargs) as response: - status_code = getattr(response, "status_code", 0) - log.debug(f"Download result: HTTP {status_code}") - if status_code != 0: - response.raise_for_status() - with open(filename, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - success = True - except RequestException as e: - log.warning(f"Failed to download {url}: {e}") - return - except AttributeError: - return - - if success: - return filename.resolve() - - -def request(self, *args, **kwargs): - """ - Multipurpose function for making web requests - - Supports custom sessions - session Request.Session() - - Arguments - cache_for (Union[None, int, float, str, datetime, timedelta]): Cache response for seconds - raise_error (bool): Whether to raise exceptions (default: False) - """ - - # we handle our own retries - retries = kwargs.pop("retries", self.config.get("http_retries", 1)) - if getattr(self, "retry_adapter", None) is None: - self.retry_adapter = HTTPAdapter(max_retries=0) - - raise_error = kwargs.pop("raise_error", False) - - cache_for = kwargs.pop("cache_for", None) - if cache_for is not None: - log.debug(f"Caching HTTP session with expire_after={cache_for}") - db_path = str(self.cache_dir / "requests-cache.sqlite") - backend = SQLiteCache(db_path=db_path) - session = CachedSession(expire_after=cache_for, backend=backend) - session.mount("http://", self.retry_adapter) - session.mount("https://", self.retry_adapter) - elif kwargs.get("session", None) is not None: - session = kwargs.pop("session", None) - session.mount("http://", self.retry_adapter) - session.mount("https://", self.retry_adapter) - else: - session = requests.Session() - session.mount("http://", self.retry_adapter) - session.mount("https://", self.retry_adapter) - - http_timeout = self.config.get("http_timeout", 20) - user_agent = self.config.get("user_agent", "BBOT") - - # in case of URL only, assume GET request - if len(args) == 1: - kwargs["url"] = args[0] - args = [] - - url = kwargs.get("url", "") - - if not args and "method" not in kwargs: - kwargs["method"] = "GET" - - if not "timeout" in kwargs: - kwargs["timeout"] = http_timeout - - headers = kwargs.get("headers", None) - - if headers is None: - headers = {} - if "User-Agent" not in headers: - headers.update({"User-Agent": user_agent}) - # only add custom headers if the URL is in-scope - if self.scan.in_scope(url): - for hk, hv in self.scan.config.get("http_headers", {}).items(): - # don't clobber headers - if hk not in headers: - headers[hk] = hv - kwargs["headers"] = headers - - http_debug = self.config.get("http_debug", False) - while retries == "infinite" or retries >= 0: - try: - if http_debug: - logstr = f"Web request: {str(args)}, {str(kwargs)}" - log.debug(logstr) - if session is not None: - response = session.request(*args, **kwargs) - else: - response = requests.request(*args, **kwargs) - if http_debug: - log.debug(f"Web response: {response} (Length: {len(response.content)}) headers: {response.headers}") - return response - except RequestException as e: - log.debug(f"Error with request: {e}") - if self.parent_helper.scan_stopping: - break - if retries != "infinite": - retries -= 1 - if retries == "infinite" or retries >= 0: - log.verbose(f'Error requesting "{url}" ({e}), retrying...') - sleep(1) - else: - if raise_error: - raise e - - class BBOTAsyncClient(httpx.AsyncClient): def __init__(self, *args, **kwargs): self._bbot_scan = kwargs.pop("_bbot_scan") @@ -217,7 +55,7 @@ def AsyncClient(self, *args, **kwargs): kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries) return BBOTAsyncClient(*args, **kwargs) - async def request_async(self, *args, **kwargs): + async def request(self, *args, **kwargs): raise_error = kwargs.pop("raise_error", False) # in case of URL only, assume GET request @@ -251,7 +89,7 @@ async def request_async(self, *args, **kwargs): if raise_error: raise - async def download_async(self, url, **kwargs): + async def download(self, url, **kwargs): """ Downloads file, returns full path of filename If download failed, returns None @@ -287,13 +125,13 @@ async def download_async(self, url, **kwargs): if success: return filename.resolve() - async def wordlist_async(self, path, lines=None, **kwargs): + async def wordlist(self, path, lines=None, **kwargs): if not path: raise WordlistError(f"Invalid wordlist: {path}") if not "cache_hrs" in kwargs: kwargs["cache_hrs"] = 720 if self.parent_helper.is_url(path): - filename = await self.download_async(str(path), **kwargs) + filename = await self.download(str(path), **kwargs) if filename is None: raise WordlistError(f"Unable to retrieve wordlist from {path}") else: @@ -314,115 +152,110 @@ async def wordlist_async(self, path, lines=None, **kwargs): f.write(line) return truncated_filename + async def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs): + page = 1 + offset = 0 + while 1: + new_url = url.format(page=page, page_size=page_size, offset=offset) + result = await self.request(new_url, **requests_kwargs) + try: + if json: + result = result.json() + yield result + except Exception: + log.warning(f'Error in api_page_iter() for url: "{new_url}"') + log.trace(traceback.format_exc()) + break + finally: + offset += page_size + page += 1 -def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs): - page = 1 - offset = 0 - while 1: - new_url = url.format(page=page, page_size=page_size, offset=offset) - result = self.request(new_url, **requests_kwargs) - try: - if json: - result = result.json() - yield result - except Exception: - import traceback - - log.warning(f'Error in api_page_iter() for url: "{new_url}"') - log.trace(traceback.format_exc()) - break - finally: - offset += page_size - page += 1 - + async def curl(self, *args, **kwargs): + url = kwargs.get("url", "") -def curl(self, *args, **kwargs): - url = kwargs.get("url", "") + if not url: + raise CurlError("No URL supplied to CURL helper") - if not url: - raise CurlError("No URL supplied to CURL helper") + curl_command = ["curl", url, "-s"] - curl_command = ["curl", url, "-s"] + raw_path = kwargs.get("raw_path", False) + if raw_path: + curl_command.append("--path-as-is") - raw_path = kwargs.get("raw_path", False) - if raw_path: - curl_command.append("--path-as-is") + # respect global ssl verify settings + ssl_verify = self.config.get("ssl_verify") + if ssl_verify == False: + curl_command.append("-k") - # respect global ssl verify settings - ssl_verify = self.config.get("ssl_verify") - if ssl_verify == False: - curl_command.append("-k") + headers = kwargs.get("headers", {}) - headers = kwargs.get("headers", {}) + ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False) - ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False) + if ignore_bbot_global_settings: + log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied") + else: + http_timeout = self.config.get("http_timeout", 20) + user_agent = self.config.get("user_agent", "BBOT") - if ignore_bbot_global_settings: - log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied") - else: - http_timeout = self.config.get("http_timeout", 20) - user_agent = self.config.get("user_agent", "BBOT") + if "User-Agent" not in headers: + headers["User-Agent"] = user_agent - if "User-Agent" not in headers: - headers["User-Agent"] = user_agent + # only add custom headers if the URL is in-scope + if self.scan.in_scope(url): + for hk, hv in self.scan.config.get("http_headers", {}).items(): + headers[hk] = hv - # only add custom headers if the URL is in-scope - if self.scan.in_scope(url): - for hk, hv in self.scan.config.get("http_headers", {}).items(): - headers[hk] = hv + # add the timeout + if not "timeout" in kwargs: + timeout = http_timeout - # add the timeout - if not "timeout" in kwargs: - timeout = http_timeout + curl_command.append("-m") + curl_command.append(str(timeout)) - curl_command.append("-m") - curl_command.append(str(timeout)) + for k, v in headers.items(): + if isinstance(v, list): + for x in v: + curl_command.append("-H") + curl_command.append(f"{k}: {x}") - for k, v in headers.items(): - if isinstance(v, list): - for x in v: + else: curl_command.append("-H") - curl_command.append(f"{k}: {x}") - - else: - curl_command.append("-H") - curl_command.append(f"{k}: {v}") - - post_data = kwargs.get("post_data", {}) - if len(post_data.items()) > 0: - curl_command.append("-d") - post_data_str = "" - for k, v in post_data.items(): - post_data_str += f"&{k}={v}" - curl_command.append(post_data_str.lstrip("&")) - - method = kwargs.get("method", "") - if method: - curl_command.append("-X") - curl_command.append(method) - - cookies = kwargs.get("cookies", "") - if cookies: - curl_command.append("-b") - cookies_str = "" - for k, v in cookies.items(): - cookies_str += f"{k}={v}; " - curl_command.append(f'{cookies_str.rstrip(" ")}') - - path_override = kwargs.get("path_override", None) - if path_override: - curl_command.append("--request-target") - curl_command.append(f"{path_override}") - - head_mode = kwargs.get("head_mode", None) - if head_mode: - curl_command.append("-I") - - raw_body = kwargs.get("raw_body", None) - if raw_body: - curl_command.append("-d") - curl_command.append(raw_body) - - output_bytes = self.run(curl_command, text=False).stdout - output = self.smart_decode(output_bytes) - return output + curl_command.append(f"{k}: {v}") + + post_data = kwargs.get("post_data", {}) + if len(post_data.items()) > 0: + curl_command.append("-d") + post_data_str = "" + for k, v in post_data.items(): + post_data_str += f"&{k}={v}" + curl_command.append(post_data_str.lstrip("&")) + + method = kwargs.get("method", "") + if method: + curl_command.append("-X") + curl_command.append(method) + + cookies = kwargs.get("cookies", "") + if cookies: + curl_command.append("-b") + cookies_str = "" + for k, v in cookies.items(): + cookies_str += f"{k}={v}; " + curl_command.append(f'{cookies_str.rstrip(" ")}') + + path_override = kwargs.get("path_override", None) + if path_override: + curl_command.append("--request-target") + curl_command.append(f"{path_override}") + + head_mode = kwargs.get("head_mode", None) + if head_mode: + curl_command.append("-I") + + raw_body = kwargs.get("raw_body", None) + if raw_body: + curl_command.append("-d") + curl_command.append(raw_body) + + output = await self.run(curl_command).stdout + return output diff --git a/bbot/modules/base.py b/bbot/modules/base.py index e4c23e9d90..3f8ebedd86 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -119,7 +119,7 @@ def handle_batch(self, *events): """ pass - def filter_event(self, event): + async def filter_event(self, event): """ Accept/reject events based on custom criteria @@ -208,7 +208,7 @@ async def catch(self, *args, **kwargs): return await self.scan.manager.catch(*args, **kwargs) async def _postcheck_and_run(self, callback, event): - acceptable, reason = self._event_postcheck(event) + acceptable, reason = await self._event_postcheck(event) if not acceptable: if reason: self.debug(f"Not accepting {event} because {reason}") @@ -411,7 +411,7 @@ def _event_precheck(self, event): return False, "module consumes IP ranges directly" return True, "" - def _event_postcheck(self, event): + async def _event_postcheck(self, event): """ Check if an event should be accepted by the module Used when taking an event FROM the module's queue (immediately before it's handled) @@ -437,7 +437,7 @@ def _event_postcheck(self, event): # custom filtering try: - filter_result = self.filter_event(event) + filter_result = await self.filter_event(event) msg = str(self._custom_filter_criteria_msg) with suppress(ValueError, TypeError): filter_result, reason = filter_result diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py index a5cc90599e..66c0f70b62 100644 --- a/bbot/modules/crobat.py +++ b/bbot/modules/crobat.py @@ -22,19 +22,19 @@ class crobat(BaseModule): # until the queue is ready to receive its results _qsize = 1 - def setup(self): + async def setup(self): self.processed = set() self.http_timeout = self.scan.config.get("http_timeout", 10) self._failures = 0 return True - def _is_wildcard(self, query): - for domain, wildcard_rdtypes in self.helpers.is_wildcard_domain(query).items(): + async def _is_wildcard(self, query): + for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query)).items(): if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")): return True return False - def filter_event(self, event): + async def filter_event(self, event): """ This filter_event is used across many modules """ @@ -42,16 +42,16 @@ def filter_event(self, event): # reject if already processed if self.already_processed(query): return False, "Event was already processed" - eligible, reason = self.eligible_for_enumeration(event) + eligible, reason = await self.eligible_for_enumeration(event) if eligible: self.processed.add(hash(query)) return True, reason return False, reason - def eligible_for_enumeration(self, event): + async def eligible_for_enumeration(self, event): query = self.make_query(event) # check if wildcard - is_wildcard = self._is_wildcard(query) + is_wildcard = await self._is_wildcard(query) # check if cloud is_cloud = False if any(t.startswith("cloud-") for t in event.tags): @@ -85,7 +85,7 @@ def abort_if(self, event): return True return False - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) results = self.query(query) if results: diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 74e67ef507..7e9d65f969 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -104,7 +104,7 @@ async def handle_event(self, event): quick=True, ) - def filter_event(self, event): + async def filter_event(self, event): # don't accept IP_RANGE --> IP_ADDRESS events from self if str(event.module) == "speculate": if not (event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE"): diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 718dc0eb58..e986206fa8 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -62,26 +62,26 @@ class massdns(crobat): digit_regex = re.compile(r"\d+") - def setup(self): + async def setup(self): self.found = dict() self.mutations_tried = set() self.source_events = dict() - self.subdomain_file = self.helpers.wordlist(self.config.get("wordlist")) + self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist")) self.max_resolvers = self.config.get("max_resolvers", 500) self.max_mutations = self.config.get("max_mutations", 500) nameservers_url = ( "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt" ) - self.resolver_file = self.helpers.wordlist( + self.resolver_file = await self.helpers.wordlist( nameservers_url, cache_hrs=24 * 7, ) self.devops_mutations = list(self.helpers.word_cloud.devops_mutations) - return super().setup() + return await super().setup() - def filter_event(self, event): + async def filter_event(self, event): query = self.make_query(event) - eligible, reason = self.eligible_for_enumeration(event) + eligible, reason = await self.eligible_for_enumeration(event) if eligible: self.add_found(event) # reject if already processed @@ -92,14 +92,14 @@ def filter_event(self, event): return True, reason return False, reason - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) h = hash(query) if not h in self.source_events: self.source_events[h] = event self.info(f"Brute-forcing subdomains for {query}") - for hostname in self.massdns(query, self.helpers.read_file(self.subdomain_file)): + for hostname in await self.massdns(query, self.helpers.read_file(self.subdomain_file)): self.emit_result(hostname, event, query) def abort_if(self, event): @@ -118,33 +118,35 @@ def already_processed(self, hostname): return True return False - def massdns(self, domain, subdomains): + async def massdns(self, domain, subdomains): abort_msg = f"Aborting massdns on {domain} due to false positives" - if self._canary_check(domain): + if await self._canary_check(domain): self.info(abort_msg) return [] - results = list(self._massdns(domain, subdomains)) + results = [l async for l in self._massdns(domain, subdomains)] if len(results) > 50: - if self._canary_check(domain): + if await self._canary_check(domain): self.info(abort_msg) return [] self.verbose(f"Resolving batch of {len(results):,} results") - resolved = dict(self.helpers.resolve_batch(results, type=("A", "AAAA", "CNAME"), cache_result=True)) + resolved = dict( + [l async for l in self.helpers.resolve_batch(results, type=("A", "AAAA", "CNAME"), cache_result=True)] + ) resolved = {k: v for k, v in resolved.items() if v} for hostname in resolved: self.add_found(hostname) return list(resolved) - def _canary_check(self, domain, num_checks=50): + async def _canary_check(self, domain, num_checks=50): random_subdomains = list(self.gen_random_subdomains(num_checks)) self.verbose(f"Testing {len(random_subdomains):,} canaries against {domain}") - canary_results = list(self._massdns(domain, random_subdomains)) + canary_results = [l async for l in self._massdns(domain, random_subdomains)] for result in canary_results: - if self.helpers.resolve(result): + if await self.helpers.resolve(result): return True return False - def _massdns(self, domain, subdomains): + async def _massdns(self, domain, subdomains): """ { "name": "www.blacklanternsecurity.com.", @@ -176,7 +178,7 @@ def _massdns(self, domain, subdomains): return domain_wildcard_rdtypes = set() - for domain, rdtypes in self.helpers.is_wildcard_domain(domain).items(): + for domain, rdtypes in (await self.helpers.is_wildcard_domain(domain)).items(): for rdtype, results in rdtypes.items(): if results: domain_wildcard_rdtypes.add(rdtype) @@ -197,7 +199,7 @@ def _massdns(self, domain, subdomains): ) subdomains = self.gen_subdomains(subdomains, domain) hosts_yielded = set() - for line in self.helpers.run_live(command, stderr=subprocess.DEVNULL, input=subdomains): + async for line in self.helpers.run_live(command, stderr=subprocess.DEVNULL, input=subdomains): try: j = json.loads(line) except json.decoder.JSONDecodeError: @@ -222,7 +224,7 @@ def _massdns(self, domain, subdomains): f"Skipping {hostname}:{rdtype} because it may be a wildcard (reason: performance)" ) continue - wildcard_rdtypes = self.helpers.is_wildcard(hostname, ips=(data,)) + wildcard_rdtypes = await self.helpers.is_wildcard(hostname, ips=(data,)) if rdtype in wildcard_rdtypes: self.debug(f"Skipping {hostname}:{rdtype} because it's a wildcard") continue @@ -231,7 +233,7 @@ def _massdns(self, domain, subdomains): hosts_yielded.add(hostname_hash) yield hostname - def finish(self): + async def finish(self): found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True) # if we have a lot of rounds to make, don't try mutations on less-populated domains trimmed_found = [] @@ -310,7 +312,7 @@ def add_mutation(_domain_hash, m): if mutations: self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})") - results = list(self.massdns(query, mutations)) + results = list(await self.massdns(query, mutations)) for hostname in results: source_event = self.get_source_event(hostname) if source_event is None: @@ -334,7 +336,7 @@ def add_found(self, host): except KeyError: self.found[domain] = set((subdomain,)) - def gen_subdomains(self, prefixes, domain): + async def gen_subdomains(self, prefixes, domain): for p in prefixes: d = f"{p}.{domain}" yield d diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 465d2f7a4a..df7bb80e15 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -193,7 +193,7 @@ async def start(self): self.warning(f"No scan targets specified") # start status ticker - ticker_task = asyncio.create_task(self._status_ticker(self.)) + ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) scan_start_time = datetime.now() try: @@ -322,7 +322,7 @@ async def setup_modules(self, remove_failed=True): hard_failed = [] soft_failed = [] - for task in asyncio.as_completed([m._setup() for m in self.modules.values()]): + for task in asyncio.as_completed([asyncio.create_task(m._setup()) for m in self.modules.values()]): module_name, status, msg = await task if status == True: self.debug(f"Setup succeeded for {module_name} ({msg})") diff --git a/bbot/test/test_step_2/test_command.py b/bbot/test/test_step_2/test_command.py new file mode 100644 index 0000000000..ad27ea1f5b --- /dev/null +++ b/bbot/test/test_step_2/test_command.py @@ -0,0 +1,63 @@ +from ..bbot_fixtures import * + + +@pytest.mark.asyncio +async def test_command(bbot_scanner, bbot_config): + scan1 = bbot_scanner(config=bbot_config) + + # run + assert "plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"])).stdout + result = (await scan1.helpers.run(["cat"], input="some\nrandom\nstdin")).stdout + assert result.splitlines() == ["some", "random", "stdin"] + + # run_live + lines = [] + async for line in scan1.helpers.run_live(["echo", "plumbus"]): + lines.append(line) + assert lines == ["plumbus"] + lines = [] + async for line in scan1.helpers.run_live(["cat"], input="some\nrandom\nstdin"): + lines.append(line) + assert lines == ["some", "random", "stdin"] + + # test piping + lines = [] + async for line in scan1.helpers.run_live( + ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"]) + ): + lines.append(line) + log.critical(lines) + assert lines == ["some", "random", "stdin"] + + # test missing executable + result = await scan1.helpers.run(["sgkjlskdfsdf"]) + assert result is None + + # test sudo + existence of environment variables + scan1.load_modules() + path_parts = os.environ.get("PATH", "").split(":") + assert "/tmp/.bbot_test/tools" in path_parts + run_lines = (await scan1.helpers.run(["env"])).stdout.splitlines() + assert f"BBOT_PLUMBUS=asdf" in run_lines + for line in run_lines: + if line.startswith("PATH="): + path_parts = line.split("=", 1)[-1].split(":") + assert "/tmp/.bbot_test/tools" in path_parts + run_lines_sudo = (await scan1.helpers.run(["env"], sudo=True)).stdout.splitlines() + assert f"BBOT_PLUMBUS=asdf" in run_lines_sudo + for line in run_lines_sudo: + if line.startswith("PATH="): + path_parts = line.split("=", 1)[-1].split(":") + assert "/tmp/.bbot_test/tools" in path_parts + run_live_lines = [l async for l in scan1.helpers.run_live(["env"])] + assert f"BBOT_PLUMBUS=asdf" in run_live_lines + for line in run_live_lines: + if line.startswith("PATH="): + path_parts = line.strip().split("=", 1)[-1].split(":") + assert "/tmp/.bbot_test/tools" in path_parts + run_live_lines_sudo = [l async for l in scan1.helpers.run_live(["env"], sudo=True)] + assert f"BBOT_PLUMBUS=asdf" in run_live_lines_sudo + for line in run_live_lines_sudo: + if line.startswith("PATH="): + path_parts = line.strip().split("=", 1)[-1].split(":") + assert "/tmp/.bbot_test/tools" in path_parts diff --git a/bbot/test/test_step_2/test_files.py b/bbot/test/test_step_2/test_files.py new file mode 100644 index 0000000000..210a6342f2 --- /dev/null +++ b/bbot/test/test_step_2/test_files.py @@ -0,0 +1,22 @@ +from time import sleep + +from ..bbot_fixtures import * + + +@pytest.mark.asyncio +async def test_files(bbot_scanner, bbot_config): + scan1 = bbot_scanner(config=bbot_config) + + # tempfile + tempfile = scan1.helpers.tempfile(("line1", "line2"), pipe=False) + assert list(scan1.helpers.read_file(tempfile)) == ["line1", "line2"] + tempfile = scan1.helpers.tempfile(("line1", "line2"), pipe=True) + assert list(scan1.helpers.read_file(tempfile)) == ["line1", "line2"] + + # tempfile tail + results = [] + tempfile = scan1.helpers.tempfile_tail(callback=lambda x: results.append(x)) + with open(tempfile, "w") as f: + f.write("asdf\n") + sleep(0.1) + assert "asdf" in results diff --git a/bbot/test/test_step_2/test_helpers.py b/bbot/test/test_step_2/test_helpers.py index 5ea38e8ff6..e3f9bc1d3b 100644 --- a/bbot/test/test_step_2/test_helpers.py +++ b/bbot/test/test_step_2/test_helpers.py @@ -2,7 +2,6 @@ import datetime import ipaddress import requests_mock -from time import sleep from ..bbot_fixtures import * @@ -351,72 +350,6 @@ def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): helpers.recursive_decode(r"Hello\\nWorld\\\tGreetings\\\\nMore\nText") == "Hello\nWorld\tGreetings\nMore\nText" ) - def raise_filenotfound(): - raise FileNotFoundError("asdf") - - def raise_brokenpipe(): - raise BrokenPipeError("asdf") - - from bbot.core.helpers import command - - command.catch(raise_filenotfound) - command.catch(raise_brokenpipe) - - ### COMMAND ### - scan1 = bbot_scanner(config=bbot_config) - assert "plumbus\n" in scan1.helpers.run(["echo", "plumbus"], text=True).stdout - assert "plumbus\n" in list(scan1.helpers.run_live(["echo", "plumbus"])) - expected_output = ["lumbus\n", "plumbus\n", "rumbus\n"] - assert list(scan1.helpers.run_live(["cat"], input="lumbus\nplumbus\nrumbus")) == expected_output - - def plumbus_generator(): - yield "lumbus" - yield "plumbus" - - assert "plumbus\n" in list(scan1.helpers.run_live(["cat"], input=plumbus_generator())) - tempfile = helpers.tempfile(("lumbus", "plumbus"), pipe=True) - with open(tempfile) as f: - assert "plumbus\n" in list(f) - tempfile = helpers.tempfile(("lumbus", "plumbus"), pipe=False) - with open(tempfile) as f: - assert "plumbus\n" in list(f) - - results = [] - tempfile = helpers.tempfile_tail(callback=lambda x: results.append(x)) - with open(tempfile, "w") as f: - f.write("asdf\n") - sleep(0.1) - assert "asdf" in results - - # test sudo + existence of environment variables - scan1.load_modules() - path_parts = os.environ.get("PATH", "").split(":") - assert "/tmp/.bbot_test/tools" in path_parts - run_lines = scan1.helpers.run(["env"]).stdout.splitlines() - assert f"BBOT_PLUMBUS=asdf" in run_lines - for line in run_lines: - if line.startswith("PATH="): - path_parts = line.split("=", 1)[-1].split(":") - assert "/tmp/.bbot_test/tools" in path_parts - run_lines_sudo = scan1.helpers.run(["env"], sudo=True).stdout.splitlines() - assert f"BBOT_PLUMBUS=asdf" in run_lines_sudo - for line in run_lines_sudo: - if line.startswith("PATH="): - path_parts = line.split("=", 1)[-1].split(":") - assert "/tmp/.bbot_test/tools" in path_parts - run_live_lines = list(scan1.helpers.run_live(["env"])) - assert f"BBOT_PLUMBUS=asdf\n" in run_live_lines - for line in run_live_lines: - if line.startswith("PATH="): - path_parts = line.strip().split("=", 1)[-1].split(":") - assert "/tmp/.bbot_test/tools" in path_parts - run_live_lines_sudo = list(scan1.helpers.run_live(["env"], sudo=True)) - assert f"BBOT_PLUMBUS=asdf\n" in run_live_lines_sudo - for line in run_live_lines_sudo: - if line.startswith("PATH="): - path_parts = line.strip().split("=", 1)[-1].split(":") - assert "/tmp/.bbot_test/tools" in path_parts - ### CACHE ### helpers.cache_put("string", "wat") helpers.cache_put("binary", b"wat") @@ -444,32 +377,6 @@ def plumbus_generator(): assert len(cache_dict) == 10 assert tuple(cache_dict) == tuple(hash(str(x)) for x in range(10, 20)) - ### WEB ### - with requests_mock.Mocker() as m: - # test base request - m.get("http://blacklanternsecurity.com/yep", text="yep") - assert getattr(helpers.request("http://blacklanternsecurity.com/yep"), "text", "") == "yep" - # test cached request - m.get("http://blacklanternsecurity.com/yepyep", text="yepyep") - assert getattr(helpers.request("http://blacklanternsecurity.com/yepyep", cache_for=60), "text", "") == "yepyep" - # test caching - m.get("http://blacklanternsecurity.com/yepyep", text="nope") - assert getattr(helpers.request("http://blacklanternsecurity.com/yepyep", cache_for=60), "text", "") == "yepyep" - # test downloading - m.get("http://blacklanternsecurity.com/download", text="downloaded") - filename = helpers.download("http://blacklanternsecurity.com/download", cache_hrs=1) - assert Path(str(filename)).is_file() - assert helpers.is_cached("http://blacklanternsecurity.com/download") - # test wordlist - m.get("http://blacklanternsecurity.com/wordlist", text="wordlist") - assert helpers.wordlist("http://blacklanternsecurity.com/wordlist").is_file() - - # custom headers - bbot_httpserver.expect_request("/test-custom-http-headers-requests", headers={"test": "header"}).respond_with_data( - "OK" - ) - assert scan.helpers.request(bbot_httpserver.url_for("/test-custom-http-headers-requests")).status_code == 200 - test_file = Path(scan.config["home"]) / "testfile.asdf" with open(test_file, "w") as f: for i in range(100): diff --git a/bbot/test/test_step_2/test_http.py b/bbot/test/test_step_2/test_http.py index 47b6c584dc..0fa4d9091e 100644 --- a/bbot/test/test_step_2/test_http.py +++ b/bbot/test/test_step_2/test_http.py @@ -17,10 +17,10 @@ async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): bbot_httpserver.expect_request(uri="/test_http_helpers", headers=headers).respond_with_data( "test_http_helpers_yep" ) - response = await scan1.helpers.request_async(url) + response = await scan1.helpers.request(url) # should fail because URL is not in-scope assert response.status_code == 500 - response = await scan2.helpers.request_async(url) + response = await scan2.helpers.request(url) # should suceed because URL is in-scope assert response.status_code == 200 assert response.text == "test_http_helpers_yep" @@ -30,7 +30,7 @@ async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): url = bbot_httpserver.url_for(path) download_content = "test_http_helpers_download_yep" bbot_httpserver.expect_request(uri=path).respond_with_data(download_content) - filename = await scan1.helpers.download_async(url) + filename = await scan1.helpers.download(url) assert Path(str(filename)).is_file() assert scan1.helpers.is_cached(url) with open(filename) as f: @@ -40,7 +40,7 @@ async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): url = bbot_httpserver.url_for(path) download_content = "404" bbot_httpserver.expect_request(uri=path).respond_with_data(download_content, status=404) - filename = await scan1.helpers.download_async(url) + filename = await scan1.helpers.download(url) assert filename is None assert not scan1.helpers.is_cached(url) @@ -49,11 +49,45 @@ async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): url = bbot_httpserver.url_for(path) download_content = "a\ncool\nword\nlist" bbot_httpserver.expect_request(uri=path).respond_with_data(download_content) - filename = await scan1.helpers.wordlist_async(url) + filename = await scan1.helpers.wordlist(url) assert Path(str(filename)).is_file() assert scan1.helpers.is_cached(url) - with open(filename) as f: - assert f.read().splitlines() == ["a", "cool", "word", "list"] + assert list(scan1.helpers.read_file(filename)) == ["a", "cool", "word", "list"] + + # page iteration + base_path = "/test_http_page_iteration" + template_path = base_path + "/{page}?page_size={page_size}&offset={offset}" + template_url = bbot_httpserver.url_for(template_path) + bbot_httpserver.expect_request( + uri=f"{base_path}/1", query_string={"page_size": "100", "offset": "0"} + ).respond_with_data("page1") + bbot_httpserver.expect_request( + uri=f"{base_path}/2", query_string={"page_size": "100", "offset": "100"} + ).respond_with_data("page2") + bbot_httpserver.expect_request( + uri=f"{base_path}/3", query_string={"page_size": "100", "offset": "200"} + ).respond_with_data("page3") + results = [] + agen = scan1.helpers.api_page_iter(template_url) + try: + async for result in agen: + if result and result.text.startswith("page"): + results.append(result) + else: + break + finally: + await agen.aclose() + assert not results + agen = scan1.helpers.api_page_iter(template_url, json=False) + try: + async for result in agen: + if result and result.text.startswith("page"): + results.append(result) + else: + break + finally: + await agen.aclose() + assert [r.text for r in results] == ["page1", "page2", "page3"] @pytest.mark.asyncio diff --git a/poetry.lock b/poetry.lock index f0fdf18237..3141b7e3fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1319,21 +1319,21 @@ files = [ [[package]] name = "requests" -version = "2.28.2" +version = "2.30.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, + {file = "requests-2.30.0-py3-none-any.whl", hash = "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294"}, + {file = "requests-2.30.0.tar.gz", hash = "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"}, ] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -1705,4 +1705,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "7d520e30d4c148bb627da6aea282645e5e3452faeb11798bef27ae81d1941f88" +content-hash = "5366646b07e8aa2c1189f64e976bab2ec624855eb5943bde2710376c8e115305" diff --git a/pyproject.toml b/pyproject.toml index cebea306f8..d8b6f50fc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ omegaconf = "^2.3.0" tldextract = "^3.4.0" psutil = "^5.9.4" wordninja = "^2.0.0" -requests = "^2.28.2" dnspython = "^2.3.0" pydantic = "^1.10.6" ansible-runner = "^2.3.2" From 0592f4b7f42c1a2b6d0f3115fc877b2fd0acd5ec Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 May 2023 17:00:32 -0400 Subject: [PATCH 008/387] better command tests --- bbot/core/helpers/command.py | 21 +++++++++++---------- bbot/core/helpers/helper.py | 2 +- bbot/test/test_step_2/test_command.py | 8 +++++++- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 65b376bf66..426d76333e 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -15,7 +15,7 @@ async def run(self, *command, **kwargs): process = await run(["ls", "/tmp"]) process.stdout --> "file1.txt\nfile2.txt" """ - proc, _input = await self._spawn_proc(*command, **kwargs) + proc, _input, command = await self._spawn_proc(*command, **kwargs) if proc is not None: if _input is not None: _input = smart_encode(_input) @@ -26,7 +26,7 @@ async def run(self, *command, **kwargs): stdout = smart_decode(stdout) if stderr and proc.returncode != 0: command_str = " ".join(command) - log.warning(f"Stderr for {command_str}:\n\t{stderr}") + log.warning(f"Stderr for run({command_str}):\n\t{stderr}") return CompletedProcess(command, proc.returncode, stdout, stderr) @@ -37,7 +37,7 @@ async def run_live(self, *command, **kwargs): async for line in run_live(["ls", "/tmp"]): log.info(line) """ - proc, _input = await self._spawn_proc(*command, **kwargs) + proc, _input, command = await self._spawn_proc(*command, **kwargs) if proc is not None: input_task = None if _input is not None: @@ -54,13 +54,14 @@ async def run_live(self, *command, **kwargs): await proc.wait() # surface stderr - # if proc.stderr and proc.returncode != 0: - # command_str = " ".join(command) - # log.warning(f"Stderr for {command_str}:\n\t{stderr}") + if proc.returncode != 0: + stdout, stderr = await proc.communicate() + command_str = " ".join(command) + log.warning(f"Stderr for run_live({command_str}):\n\t{smart_decode(stderr)}") async def _spawn_proc(self, *command, **kwargs): - command, kwargs = self._prepare_command_kwargs_async(command, kwargs) + command, kwargs = self._prepare_command_kwargs(command, kwargs) _input = kwargs.pop("input", None) if _input is not None: if kwargs.get("stdin") is not None: @@ -70,11 +71,11 @@ async def _spawn_proc(self, *command, **kwargs): log.hugeverbose(f"run: {' '.join(command)}") try: proc = await asyncio.create_subprocess_exec(*command, **kwargs) - return proc, _input + return proc, _input, command except FileNotFoundError as e: log.warning(f"{e} - missing executable?") log.trace(traceback.format_exc()) - return None, None + return None, None, None async def _write_stdin(proc, _input): @@ -88,7 +89,7 @@ async def _write_stdin(proc, _input): proc.stdin.close() -def _prepare_command_kwargs_async(self, command, kwargs): +def _prepare_command_kwargs(self, command, kwargs): if not "stdout" in kwargs: kwargs["stdout"] = asyncio.subprocess.PIPE if not "stderr" in kwargs: diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index cff12d7848..f25578d30b 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -23,8 +23,8 @@ class ConfigAwareHelper: from . import regexes from . import validators from .files import tempfile, feed_pipe, _feed_pipe, tempfile_tail + from .command import run, run_live, _spawn_proc, _prepare_command_kwargs from .cache import cache_get, cache_put, cache_filename, is_cached, CacheDict - from .command import run, run_live, _spawn_proc, _prepare_command_kwargs_async def __init__(self, config, scan=None): self.config = config diff --git a/bbot/test/test_step_2/test_command.py b/bbot/test/test_step_2/test_command.py index ad27ea1f5b..db2a069290 100644 --- a/bbot/test/test_step_2/test_command.py +++ b/bbot/test/test_step_2/test_command.py @@ -26,12 +26,18 @@ async def test_command(bbot_scanner, bbot_config): ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"]) ): lines.append(line) - log.critical(lines) assert lines == ["some", "random", "stdin"] # test missing executable result = await scan1.helpers.run(["sgkjlskdfsdf"]) assert result is None + lines = [l async for l in scan1.helpers.run_live(["ljhsdghsdf"])] + assert not lines + # test stderr + result = await scan1.helpers.run(["ls", "/sldikgjasldkfsdf"]) + assert "No such file or directory" in result.stderr + lines = [l async for l in scan1.helpers.run_live(["ls", "/sldikgjasldkfsdf"])] + assert not lines # test sudo + existence of environment variables scan1.load_modules() From 65080555b707f6e2b06d8302cc49bdd4460a91a9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 8 May 2023 17:08:50 -0400 Subject: [PATCH 009/387] asyncified curl --- bbot/core/helpers/web.py | 12 +++--- bbot/test/test_step_1/test_before_patching.py | 34 ----------------- .../test_step_2/{test_http.py => test_web.py} | 37 ++++++++++++++++++- 3 files changed, 41 insertions(+), 42 deletions(-) delete mode 100644 bbot/test/test_step_1/test_before_patching.py rename bbot/test/test_step_2/{test_http.py => test_web.py} (72%) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 3d44e298cc..ba642d3554 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -183,7 +183,7 @@ async def curl(self, *args, **kwargs): curl_command.append("--path-as-is") # respect global ssl verify settings - ssl_verify = self.config.get("ssl_verify") + ssl_verify = self.parent_helper.config.get("ssl_verify") if ssl_verify == False: curl_command.append("-k") @@ -194,15 +194,15 @@ async def curl(self, *args, **kwargs): if ignore_bbot_global_settings: log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied") else: - http_timeout = self.config.get("http_timeout", 20) - user_agent = self.config.get("user_agent", "BBOT") + http_timeout = self.parent_helper.config.get("http_timeout", 20) + user_agent = self.parent_helper.config.get("user_agent", "BBOT") if "User-Agent" not in headers: headers["User-Agent"] = user_agent # only add custom headers if the URL is in-scope - if self.scan.in_scope(url): - for hk, hv in self.scan.config.get("http_headers", {}).items(): + if self.parent_helper.scan.in_scope(url): + for hk, hv in self.parent_helper.scan.config.get("http_headers", {}).items(): headers[hk] = hv # add the timeout @@ -257,5 +257,5 @@ async def curl(self, *args, **kwargs): curl_command.append("-d") curl_command.append(raw_body) - output = await self.run(curl_command).stdout + output = (await self.parent_helper.run(curl_command)).stdout return output diff --git a/bbot/test/test_step_1/test_before_patching.py b/bbot/test/test_step_1/test_before_patching.py deleted file mode 100644 index 5fe26b0fba..0000000000 --- a/bbot/test/test_step_1/test_before_patching.py +++ /dev/null @@ -1,34 +0,0 @@ -from ..bbot_fixtures import * # noqa: F401 -from bbot.scanner import Scanner - - -def test_curl(bbot_httpserver, bbot_config): - scan = Scanner("127.0.0.1", config=bbot_config) - helpers = scan.helpers - url = bbot_httpserver.url_for("/curl") - bbot_httpserver.expect_request(uri="/curl").respond_with_data("curl_yep") - bbot_httpserver.expect_request(uri="/index.html").respond_with_data("curl_yep_index") - assert helpers.curl(url=url) == "curl_yep" - assert helpers.curl(url=url, ignore_bbot_global_settings=True) == "curl_yep" - assert helpers.curl(url=url, head_mode=True).startswith("HTTP/") - assert helpers.curl(url=url, raw_body="body") == "curl_yep" - assert ( - helpers.curl( - url=url, - raw_path=True, - headers={"test": "test", "test2": ["test2"]}, - ignore_bbot_global_settings=False, - post_data={"test": "test"}, - method="POST", - cookies={"test": "test"}, - path_override="/index.html", - ) - == "curl_yep_index" - ) - # test custom headers - bbot_httpserver.expect_request("/test-custom-http-headers-curl", headers={"test": "header"}).respond_with_data( - "curl_yep_headers" - ) - headers_url = bbot_httpserver.url_for("/test-custom-http-headers-curl") - curl_result = helpers.curl(url=headers_url) - assert curl_result == "curl_yep_headers" diff --git a/bbot/test/test_step_2/test_http.py b/bbot/test/test_step_2/test_web.py similarity index 72% rename from bbot/test/test_step_2/test_http.py rename to bbot/test/test_step_2/test_web.py index 0fa4d9091e..323abf8c6f 100644 --- a/bbot/test/test_step_2/test_http.py +++ b/bbot/test/test_step_2/test_web.py @@ -2,7 +2,7 @@ @pytest.mark.asyncio -async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): +async def test_web_helpers(bbot_scanner, bbot_config, bbot_httpserver): scan1 = bbot_scanner("8.8.8.8", config=bbot_config) scan2 = bbot_scanner("127.0.0.1", config=bbot_config) @@ -91,7 +91,7 @@ async def test_http_helpers(bbot_scanner, bbot_config, bbot_httpserver): @pytest.mark.asyncio -async def test_http_interactsh(bbot_scanner, bbot_config, bbot_httpserver): +async def test_web_interactsh(bbot_scanner, bbot_config, bbot_httpserver): from bbot.core.helpers.interactsh import server_list scan1 = bbot_scanner("8.8.8.8", config=bbot_config) @@ -106,3 +106,36 @@ async def async_callback(data): data_list = await interactsh_client.poll() assert isinstance(data_list, list) assert await interactsh_client.deregister() is None + + +@pytest.mark.asyncio +async def test_web_curl(bbot_scanner, bbot_config, bbot_httpserver): + scan = bbot_scanner("127.0.0.1", config=bbot_config) + helpers = scan.helpers + url = bbot_httpserver.url_for("/curl") + bbot_httpserver.expect_request(uri="/curl").respond_with_data("curl_yep") + bbot_httpserver.expect_request(uri="/index.html").respond_with_data("curl_yep_index") + assert await helpers.curl(url=url) == "curl_yep" + assert await helpers.curl(url=url, ignore_bbot_global_settings=True) == "curl_yep" + assert (await helpers.curl(url=url, head_mode=True)).startswith("HTTP/") + assert await helpers.curl(url=url, raw_body="body") == "curl_yep" + assert ( + await helpers.curl( + url=url, + raw_path=True, + headers={"test": "test", "test2": ["test2"]}, + ignore_bbot_global_settings=False, + post_data={"test": "test"}, + method="POST", + cookies={"test": "test"}, + path_override="/index.html", + ) + == "curl_yep_index" + ) + # test custom headers + bbot_httpserver.expect_request("/test-custom-http-headers-curl", headers={"test": "header"}).respond_with_data( + "curl_yep_headers" + ) + headers_url = bbot_httpserver.url_for("/test-custom-http-headers-curl") + curl_result = await helpers.curl(url=headers_url) + assert curl_result == "curl_yep_headers" From 229c381fc0b0b1d4ddf403b112746ef192a69dd6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 10 May 2023 14:01:05 -0400 Subject: [PATCH 010/387] httpx working --- bbot/agent/agent.py | 2 +- bbot/cli.py | 9 +- bbot/core/errors.py | 6 +- bbot/core/helpers/command.py | 22 ++- bbot/core/helpers/dns.py | 42 +++--- bbot/core/helpers/interactsh.py | 3 +- bbot/core/helpers/misc.py | 21 +++ bbot/core/logger/logger.py | 1 + bbot/modules/base.py | 184 +++++++++++--------------- bbot/modules/httpx.py | 8 +- bbot/modules/internal/speculate.py | 4 +- bbot/modules/massdns.py | 9 +- bbot/modules/naabu.py | 10 +- bbot/modules/paramminer_cookies.py | 3 - bbot/modules/paramminer_getparams.py | 3 - bbot/modules/paramminer_headers.py | 6 +- bbot/modules/report/asn.py | 3 - bbot/scanner/manager.py | 130 +++++------------- bbot/scanner/scanner.py | 168 ++++++++++++++++------- bbot/test/test_step_2/test_command.py | 6 + bbot/test/test_step_2/test_manager.py | 21 +-- 21 files changed, 329 insertions(+), 332 deletions(-) diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py index be61e2d129..3076119bef 100644 --- a/bbot/agent/agent.py +++ b/bbot/agent/agent.py @@ -137,7 +137,7 @@ def stop_scan(self): log.warning(msg) return {"error": msg} scan_id = str(self.scan.id) - self.scan.stop(wait=True) + self.scan.stop() msg = f"Stopped scan {scan_id}" log.warning(msg) self.scan = None diff --git a/bbot/cli.py b/bbot/cli.py index c1c637839f..8538e269d2 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -320,12 +320,9 @@ def keyboard_listen(): os._exit(1) # debug troublesome modules - """ - from time import sleep - while 1: - scanner.manager.modules_status(_log=True) - sleep(1) - """ + # while 1: + # await scanner.manager.modules_status(_log=True) + # await asyncio.sleep(1) def main(): diff --git a/bbot/core/errors.py b/bbot/core/errors.py index ee93ee6065..24c94d4343 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -1,4 +1,4 @@ -from requests.exceptions import RequestException # noqa F401 +from httpx import RequestError # noqa class BBOTError(Exception): @@ -9,10 +9,6 @@ class ScanError(BBOTError): pass -class ScanCancelledError(BBOTError): - pass - - class ValidationError(BBOTError): pass diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 426d76333e..219fa1bc73 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -18,7 +18,10 @@ async def run(self, *command, **kwargs): proc, _input, command = await self._spawn_proc(*command, **kwargs) if proc is not None: if _input is not None: - _input = smart_encode(_input) + if isinstance(_input, (list, tuple)): + _input = b"\n".join(smart_encode(i) for i in _input) + b"\n" + else: + _input = smart_encode(_input) stdout, stderr = await proc.communicate(_input) # surface stderr @@ -50,14 +53,18 @@ async def run_live(self, *command, **kwargs): yield smart_decode(line).rstrip("\r\n") if input_task is not None: - await input_task + try: + await input_task + except BrokenPipeError: + log.trace(traceback.format_exc()) await proc.wait() # surface stderr if proc.returncode != 0: stdout, stderr = await proc.communicate() - command_str = " ".join(command) - log.warning(f"Stderr for run_live({command_str}):\n\t{smart_decode(stderr)}") + if stderr: + command_str = " ".join(command) + log.warning(f"Stderr for run_live({command_str}):\n\t{smart_decode(stderr)}") async def _spawn_proc(self, *command, **kwargs): @@ -80,8 +87,11 @@ async def _spawn_proc(self, *command, **kwargs): async def _write_stdin(proc, _input): if _input is not None: - if isinstance(_input, str): - proc.stdin.write(smart_encode(_input)) + if isinstance(_input, (str, bytes)): + _input = [_input] + if isinstance(_input, (list, tuple)): + for chunk in _input: + proc.stdin.write(smart_encode(chunk) + b"\n") else: async for chunk in _input: proc.stdin.write(smart_encode(chunk) + b"\n") diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 56e24262fd..2e1bd093fd 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -72,11 +72,13 @@ async def resolve(self, query, **kwargs): } """ results = set() - raw_results, errors = await self.resolve_raw(query, **kwargs) - for rdtype, answers in raw_results: - for answer in answers: - for _, t in self.extract_targets(answer): - results.add(t) + r = await self.resolve_raw(query, **kwargs) + if r: + raw_results, errors = r + for rdtype, answers in raw_results: + for answer in answers: + for _, t in self.extract_targets(answer): + results.add(t) return results async def resolve_raw(self, query, **kwargs): @@ -350,22 +352,24 @@ def event_cache_get(self, host): except KeyError: return set(), None, None, set() + async def _resolve_batch_coro_wrapper(self, q, **kwargs): + """ + Helps us correlate task results back to their original arguments + """ + result = await self.resolve(q, **kwargs) + return (q, result) + async def resolve_batch(self, queries, **kwargs): """ - await resolve_batch("www.evilcorp.com", "evilcorp.com") --> [ + await resolve_batch(["www.evilcorp.com", "evilcorp.com"]) --> [ ("www.evilcorp.com", {"1.1.1.1"}), ("evilcorp.com", {"2.2.2.2"}) ] """ - async def coro_wrapper(q, **_kwargs): - """ - Helps us correlate task results back to their original arguments - """ - result = await self.resolve(q, **_kwargs) - return (q, result) - - for task in asyncio.as_completed([coro_wrapper(q, **kwargs) for q in queries]): + for task in asyncio.as_completed( + [asyncio.create_task(self._resolve_batch_coro_wrapper(q, **kwargs)) for q in queries] + ): yield await task def extract_targets(self, record): @@ -439,10 +443,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None): # skip check if the query is a domain if is_domain(query): return {} - # skip check if the query's parent domain is excluded in the config - for d in self.wildcard_ignore: - if self.parent_helper.host_in_host(query, d): - return {} parent = parent_domain(query) parents = list(domain_parents(query)) @@ -513,6 +513,12 @@ async def is_wildcard_domain(self, domain, log_info=False): wildcard_domain_results = {} domain = self._clean_dns_record(domain) + # skip check if the query's parent domain is excluded in the config + for d in self.wildcard_ignore: + if self.parent_helper.host_in_host(domain, d): + log.debug(f"Skipping wildcard detection on {domain} because it is excluded in the config") + return {} + # make a list of its parents parents = list(domain_parents(domain, include_self=True)) # and check each of them, beginning with the highest parent (i.e. the root domain) diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index f3303960f4..6cb679b4e9 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -126,7 +126,8 @@ async def poll(self): return ret async def poll_loop(self, callback): - return await self.parent_helper.scan.manager.catch(self._poll_loop, callback, _force=True) + async with self.parent_helper.scan.acatch(context=self._poll_loop): + return await self._poll_loop(callback) async def _poll_loop(self, callback): while 1: diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 8395d285a0..5e95899200 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1031,3 +1031,24 @@ async def execute_sync_or_async(callback, *args, **kwargs): return await callback(*args, **kwargs) else: return callback(*args, **kwargs) + + +def get_exception_chain(e): + """ + Get the full chain of exceptions that led to the current one + """ + exception_chain = [] + current_exception = e + while current_exception is not None: + exception_chain.append(current_exception) + current_exception = getattr(current_exception, "__context__", None) + return exception_chain + + +def get_traceback_details(e): + tb = traceback.extract_tb(e.__traceback__) + last_frame = tb[-1] # Get the last frame in the traceback (the one where the exception was raised) + filename = last_frame.filename + lineno = last_frame.lineno + funcname = last_frame.name + return filename, lineno, funcname diff --git a/bbot/core/logger/logger.py b/bbot/core/logger/logger.py index c5668b1d89..e7ffe96628 100644 --- a/bbot/core/logger/logger.py +++ b/bbot/core/logger/logger.py @@ -221,6 +221,7 @@ def set_log_level(level, logger=None): _log_level_override = level log = logging.getLogger("bbot") log.setLevel(level) + logging.getLogger("asyncio").setLevel(level) def toggle_log_level(logger=None): diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 3f8ebedd86..75c9f08c40 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -7,7 +7,7 @@ from ..core.helpers.misc import get_size from ..core.helpers.async_helpers import TaskCounter -from ..core.errors import ScanCancelledError, ValidationError, WordlistError +from ..core.errors import ValidationError, WordlistError class BaseModule: @@ -98,6 +98,8 @@ def __init__(self, scan): # track number of failures (for .request_with_fail_count()) self._request_failures = 0 + self._event_received = asyncio.Condition() + async def setup(self): """ Perform setup functions at the beginning of the scan. @@ -200,56 +202,26 @@ def get_watched_events(self): self._watched_events = set(self.watched_events) return self._watched_events - def submit_task(self, *args, **kwargs): - kwargs["_block"] = False - return self.thread_pool.submit_task(self.catch, *args, **kwargs) - - async def catch(self, *args, **kwargs): - return await self.scan.manager.catch(*args, **kwargs) - - async def _postcheck_and_run(self, callback, event): - acceptable, reason = await self._event_postcheck(event) - if not acceptable: - if reason: - self.debug(f"Not accepting {event} because {reason}") - return - self.scan.stats.event_consumed(event, self) - return await callback(event) - - async def _register_running(self, callback, *args, **kwargs): - with self._task_counter: - return await callback(*args, **kwargs) - - def _handle_batch(self, force=False): + async def _handle_batch(self): + submitted = False if self.batch_size <= 1: return - if self.num_queued_events > 0 and (force or self.num_queued_events >= self.batch_size): - on_finish_callback = None - events, finish, report = self.events_waiting - if finish: - on_finish_callback = self.finish - elif report: - on_finish_callback = self.report - checked_events = [] - for e in events: - acceptable, reason = self._event_postcheck(e) - if not acceptable: - if reason: - self.debug(f"Not accepting {e} because {reason}") - continue - checked_events.append(e) - if checked_events: + if self.num_queued_events > 0: + events, finish, report = await self.events_waiting() + if not self.errored: self.debug(f"Handling batch of {len(events):,} events") - if not self.errored: - self._internal_thread_pool.submit_task( - self.catch, - self._register_running, - self.handle_batch, - *checked_events, - _on_finish_callback=on_finish_callback, - ) - return True - return False + if events: + submitted = True + async with self.scan.acatch(context=f"{self.name}.handle_batch"): + with self._task_counter: + await self.handle_batch(*events) + if finish: + async with self.scan.acatch(context=f"{self.name}.finish"): + await self.finish() + elif report: + async with self.scan.acatch(context=f"{self.name}.report"): + await self.report() + return submitted def make_event(self, *args, **kwargs): raise_error = kwargs.pop("raise_error", False) @@ -271,8 +243,7 @@ def emit_event(self, *args, **kwargs): event = self.make_event(*args, **event_kwargs) self.scan.manager.queue_event(event) - @property - def events_waiting(self): + async def events_waiting(self): """ yields all events in queue, up to maximum batch size """ @@ -284,11 +255,16 @@ def events_waiting(self): break try: event = self.incoming_event_queue.get_nowait() - if event.type == "FINISHED": - finish = True - else: - events.append(event) - except queue.Empty: + self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") + acceptable, reason = await self._event_postcheck(event) + if acceptable: + if event.type == "FINISHED": + finish = True + else: + events.append(event) + elif reason: + self.debug(f"Not accepting {event} because {reason}") + except asyncio.queues.QueueEmpty: break return events, finish, report @@ -323,53 +299,41 @@ async def _setup(self): self.trace() return self.name, status, str(msg) - @property - def _force_batch(self): - """ - Determine whether a batch should be forcefully submitted - """ - # if we're below our maximum threading potential - return self._internal_thread_pool.num_tasks < self.max_event_handlers - async def _worker(self): - try: - while not self.scan.stopping: - # hold the reigns if our outgoing queue is full - # if self._qsize and self.outgoing_event_queue.qsize() >= self._qsize: - # with self.event_received: - # await self.event_received.wait() - - if self.batch_size > 1: - pass - # submitted = self._handle_batch(force=self._force_batch) - # if not submitted: - # with self.event_received: - # await self.event_received.wait() - - else: - try: - if self.incoming_event_queue: - e = await self.incoming_event_queue.get() - else: - self.debug(f"Event queue is in bad state") - return - except queue.Empty: - continue - self.debug(f"Got {e} from {getattr(e, 'module', 'unknown_module')}") - # if we receive the special "FINISHED" event - if e.type == "FINISHED": - await self.catch(self._register_running, self.finish) - else: - await self.catch(self._register_running, self._postcheck_and_run, self.handle_event, e) + while not self.scan.stopping: + # hold the reigns if our outgoing queue is full + # if self._qsize and self.outgoing_event_queue.qsize() >= self._qsize: + # with self.event_received: + # await self.event_received.wait() + + if self.batch_size > 1: + submitted = await self._handle_batch() + if not submitted: + async with self._event_received: + await self._event_received.wait() - except KeyboardInterrupt: - self.debug(f"Interrupted") - self.scan.stop() - except ScanCancelledError as e: - self.verbose(f"Scan cancelled, {e}") - except Exception as e: - self.set_error_state(f"Exception ({e.__class__.__name__}) in module {self.name}:\n{e}") - self.trace() + else: + try: + if self.incoming_event_queue: + event = await self.incoming_event_queue.get() + else: + self.debug(f"Event queue is in bad state") + return + except asyncio.queues.QueueEmpty: + continue + self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") + acceptable, reason = await self._event_postcheck(event) + if not acceptable: + self.debug(f"Not accepting {event} because {reason}") + if acceptable: + if event.type == "FINISHED": + async with self.scan.acatch(context=f"{self.name}.finish"): + with self._task_counter: + await self.finish() + else: + async with self.scan.acatch(context=f"{self.name}.handle_event"): + with self._task_counter: + await self.handle_event(event) @property def max_scope_distance(self): @@ -416,12 +380,16 @@ async def _event_postcheck(self, event): Check if an event should be accepted by the module Used when taking an event FROM the module's queue (immediately before it's handled) """ + # special exception for "FINISHED" event if event.type in ("FINISHED",): return True, "" + # reject out-of-scope events for active modules + # TODO: reconsider this if "active" in self.flags and "target" in event.tags and event not in self.scan.whitelist: return False, "it is not in whitelist and module has active flag" + # check scope distance if self._type != "output": if self.in_scope_only: if event.scope_distance > 0: @@ -436,7 +404,7 @@ async def _event_postcheck(self, event): ) # custom filtering - try: + async with self.scan.acatch(context=self.filter_event): filter_result = await self.filter_event(event) msg = str(self._custom_filter_criteria_msg) with suppress(ValueError, TypeError): @@ -444,16 +412,12 @@ async def _event_postcheck(self, event): msg += f": {reason}" if not filter_result: return False, msg - except ScanCancelledError: - return False, "Scan cancelled" - except Exception as e: - self.error(f"Error in filter_event({event}): {e}") - self.trace() if self._type == "output" and not event._stats_recorded: event._stats_recorded = True self.scan.stats.event_produced(event) + self.debug(f"{event} passed post-check") return True, "" async def _cleanup(self): @@ -461,9 +425,11 @@ async def _cleanup(self): self._cleanedup = True for callback in [self.cleanup] + self.cleanup_callbacks: if callable(callback): - await self.catch(self._register_running, callback, _force=True) + async with self.scan.acatch(context=self.name): + with self._task_counter: + await self.helpers.execute_sync_or_async(callback) - def queue_event(self, event): + async def queue_event(self, event): if self.incoming_event_queue in (None, False): self.debug(f"Not in an acceptable state to queue event") return @@ -474,6 +440,8 @@ def queue_event(self, event): return try: self.incoming_event_queue.put_nowait(event) + async with self._event_received: + self._event_received.notify() except AttributeError: self.debug(f"Not in an acceptable state to queue event") diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 6641b38301..16ca7294c2 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -32,7 +32,7 @@ class httpx(BaseModule): scope_distance_modifier = 1 _priority = 2 - def setup(self): + async def setup(self): self.threads = self.config.get("threads", 50) self.timeout = self.scan.config.get("httpx_timeout", 5) self.retries = self.scan.config.get("httpx_retries", 1) @@ -40,7 +40,7 @@ def setup(self): self.visited = set() return True - def filter_event(self, event): + async def filter_event(self, event): if "_wildcard" in str(event.host).split("."): return False, "event is wildcard" @@ -62,7 +62,7 @@ def filter_event(self, event): # note: speculate makes open ports from return True - def handle_batch(self, *events): + async def handle_batch(self, *events): stdin = {} for e in events: url_hash = None @@ -106,7 +106,7 @@ def handle_batch(self, *events): proxy = self.scan.config.get("http_proxy", "") if proxy: command += ["-http-proxy", proxy] - for line in self.helpers.run_live(command, input=list(stdin), stderr=subprocess.DEVNULL): + async for line in self.helpers.run_live(command, input=list(stdin), stderr=subprocess.DEVNULL): try: j = json.loads(line) except json.decoder.JSONDecodeError: diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 7e9d65f969..079f8076ed 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -29,6 +29,7 @@ async def setup(self): self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()]) self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()]) self.range_to_ip = True + self.dns_resolution = self.scan.config.get("dns_resolution", True) self.ports = self.config.get("ports", [80, 443]) if isinstance(self.ports, int): @@ -88,10 +89,9 @@ async def handle_event(self, event): # from hosts if emit_open_ports: # don't act on unresolved DNS_NAMEs - usable_dns = False if event.type == "DNS_NAME": - if "a-record" in event.tags or "aaaa-record" in event.tags: + if (not self.dns_resolution) or ("a-record" in event.tags or "aaaa-record" in event.tags): usable_dns = True if event.type == "IP_ADDRESS" or usable_dns: diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index e986206fa8..addef9fc3e 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -98,7 +98,7 @@ async def handle_event(self, event): if not h in self.source_events: self.source_events[h] = event - self.info(f"Brute-forcing subdomains for {query}") + self.info(f"Brute-forcing subdomains for {query} ({event})") for hostname in await self.massdns(query, self.helpers.read_file(self.subdomain_file)): self.emit_result(hostname, event, query) @@ -141,9 +141,12 @@ async def _canary_check(self, domain, num_checks=50): random_subdomains = list(self.gen_random_subdomains(num_checks)) self.verbose(f"Testing {len(random_subdomains):,} canaries against {domain}") canary_results = [l async for l in self._massdns(domain, random_subdomains)] - for result in canary_results: - if await self.helpers.resolve(result): + async for result in self.helpers.resolve_batch(canary_results): + if result: return True + # for result in canary_results: + # if await self.helpers.resolve(result): + # return True return False async def _massdns(self, domain, subdomains): diff --git a/bbot/modules/naabu.py b/bbot/modules/naabu.py index d806ab7082..f81424df89 100644 --- a/bbot/modules/naabu.py +++ b/bbot/modules/naabu.py @@ -20,7 +20,7 @@ class naabu(BaseModule): "skip_host_discovery": "skip host discovery (-Pn)", "version": "naabu version", } - max_event_handlers = 2 + max_event_handlers = 1 batch_size = 256 _priority = 2 @@ -56,15 +56,15 @@ class naabu(BaseModule): }, ] - def setup(self): + async def setup(self): self.helpers.depsinstaller.ensure_root(message="Naabu requires root privileges") self.skip_host_discovery = self.config.get("skip_host_discovery", True) return True - def handle_batch(self, *events): + async def handle_batch(self, *events): _input = [str(e.data) for e in events] command = self.construct_command() - for line in self.helpers.run_live(command, input=_input, stderr=subprocess.DEVNULL, sudo=False): + async for line in self.helpers.run_live(command, input=_input, stderr=subprocess.DEVNULL, sudo=False): try: j = json.loads(line) except Exception as e: @@ -115,6 +115,6 @@ def construct_command(self): command += ["-top-ports", top_ports] return command - def cleanup(self): + async def cleanup(self): resume_file = self.helpers.current_dir / "resume.cfg" resume_file.unlink(missing_ok=True) diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index be2318dc4c..64f3965191 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -1,4 +1,3 @@ -from bbot.core.errors import ScanCancelledError from .paramminer_headers import paramminer_headers @@ -22,8 +21,6 @@ class paramminer_cookies(paramminer_headers): compare_mode = "cookie" def check_batch(self, compare_helper, url, cookie_list): - if self.scan.stopping: - raise ScanCancelledError() cookies = {p: self.rand_string(14) for p in cookie_list} return compare_helper.compare(url, cookies=cookies) diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 688f394628..7ad778c0e4 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -1,4 +1,3 @@ -from bbot.core.errors import ScanCancelledError from .paramminer_headers import paramminer_headers @@ -21,8 +20,6 @@ class paramminer_getparams(paramminer_headers): compare_mode = "getparam" def check_batch(self, compare_helper, url, getparam_list): - if self.scan.stopping: - raise ScanCancelledError() test_getparams = {p: self.rand_string(14) for p in getparam_list} return compare_helper.compare(self.helpers.add_get_params(url, test_getparams).geturl()) diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 0b0678825b..1009a4a33f 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -1,5 +1,5 @@ from bbot.modules.base import BaseModule -from bbot.core.errors import HttpCompareError, ScanCancelledError +from bbot.core.errors import HttpCompareError class paramminer_headers(BaseModule): @@ -70,8 +70,6 @@ def handle_event(self, event): ) results.clear() assert False - except ScanCancelledError: - return except AssertionError: pass @@ -130,8 +128,6 @@ def binary_search(self, compare_helper, url, group, reasons=None, reflection=Fal self.warning(f"Submitted group of size 0 to binary_search()") def check_batch(self, compare_helper, url, header_list): - if self.scan.stopping: - raise ScanCancelledError() rand = self.rand_string() test_headers = {} for header in header_list: diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 331fddafd1..592ec1ff63 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -1,4 +1,3 @@ -from bbot.core.errors import ScanCancelledError from bbot.modules.report.base import BaseReportModule @@ -90,8 +89,6 @@ def get_asn(self, ip, retries=1): """ for attempt in range(retries + 1): for i, source in enumerate(list(self.sources)): - if self.scan.stopping: - raise ScanCancelledError() get_asn_fn = getattr(self, f"get_asn_{source}") res = get_asn_fn(ip) if res == False: diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index e2bf595064..65a123f3e4 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -4,9 +4,9 @@ import traceback from contextlib import suppress +from ..core.errors import ValidationError from ..core.helpers.queueing import EventQueue from ..core.helpers.async_helpers import TaskCounter -from ..core.errors import ScanCancelledError, ValidationError log = logging.getLogger("bbot.scanner.manager") @@ -32,16 +32,14 @@ async def init_events(self): """ seed scanner with target events """ - with self._task_counter: - self.distribute_event(self.scan.root_event) - sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) - for event in sorted_events: - self.scan.verbose(f"Target: {event}") - self.queue_event(event) - # force submit batches - # for mod in self.scan.modules.values(): - # mod._handle_batch(force=True) - self.scan._finished_init = True + async with self.scan.acatch(context=self.init_events): + with self._task_counter: + await self.distribute_event(self.scan.root_event) + sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data)) + for event in sorted_events: + self.scan.verbose(f"Target: {event}") + self.queue_event(event) + self.scan._finished_init = True async def emit_event(self, event, *args, **kwargs): """ @@ -56,38 +54,20 @@ async def emit_event(self, event, *args, **kwargs): event._resolved.set() return False + log.debug(f'Module "{event.module}" raised {event}') + # "quick" queues the event immediately quick = kwargs.pop("quick", False) if quick: log.debug(f'Module "{event.module}" raised {event}') event._resolved.set() - for kwarg in ["abort_if", "on_success_callback", "_block"]: + for kwarg in ["abort_if", "on_success_callback"]: kwargs.pop(kwarg, None) - try: - self.distribute_event(event, *args, **kwargs) - return True - except ScanCancelledError: - return False - except Exception as e: - log.error(f"Unexpected error in manager.emit_event(): {e}") - log.trace(traceback.format_exc()) + async with self.scan.acatch(context=self.distribute_event): + await self.distribute_event(event, *args, **kwargs) else: - # don't raise an exception if the thread pool has been shutdown - error = True - try: - await self.catch(self._emit_event, event, *args, **kwargs) - error = False - log.debug(f'Module "{event.module}" raised {event}') - return True - except ScanCancelledError: - return False - except Exception as e: - log.error(f"Unexpected error in manager.emit_event(): {e}") - log.trace(traceback.format_exc()) - finally: - if error: - event._resolved.set() - return False + async with self.scan.acatch(context=self._emit_event, finally_callback=event._resolved.set): + await self._emit_event(event, *args, **kwargs) def _event_precheck(self, event, exclude=("DNS_NAME",)): """ @@ -203,7 +183,8 @@ async def _emit_event(self, event, *args, **kwargs): # now that the event is properly tagged, we can finally make decisions about it if callable(abort_if): - abort_result = abort_if(event) + async with self.scan.acatch(context=abort_if): + abort_result = await self.helpers.execute_sync_or_async(abort_if, event) msg = f"{event.module}: not raising event {event} due to custom criteria in abort_if()" with suppress(ValueError, TypeError): abort_result, reason = abort_result @@ -218,7 +199,8 @@ async def _emit_event(self, event, *args, **kwargs): # run success callback before distributing event (so it can add tags, etc.) if distribute_event: if callable(on_success_callback): - self.catch(on_success_callback, event) + async with self.scan.acatch(context=on_success_callback): + await self.scan.helpers.execute_sync_or_async(on_success_callback, event) if not event.host or (event.always_emit and not event_is_duplicate): log.debug( @@ -229,7 +211,7 @@ async def _emit_event(self, event, *args, **kwargs): self.queue_event(s) if distribute_event: - self.distribute_event(event) + await self.distribute_event(event) event_distributed = True # speculate DNS_NAMES and IP_ADDRESSes from other event types @@ -277,9 +259,6 @@ async def _emit_event(self, event, *args, **kwargs): for child_event in dns_child_events: self.queue_event(child_event) - except KeyboardInterrupt: - self.scan.stop() - except ValidationError as e: log.warning(f"Event validation failed with args={args}, kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) @@ -322,66 +301,29 @@ def accept_event(self, event): return False return True - async def catch(self, callback, *args, **kwargs): - """ - Wrapper to ensure error messages get surfaced to the user - """ - ret = None - on_finish_callback = kwargs.pop("_on_finish_callback", None) - force = kwargs.pop("_force", False) - fn = callback - for arg in args: - if callable(arg): - fn = arg - else: - break - try: - if not self.scan.stopping or force: - ret = await self.scan.helpers.execute_sync_or_async(callback, *args, **kwargs) - except ScanCancelledError as e: - log.debug(f"ScanCancelledError in {fn.__qualname__}(): {e}") - except BrokenPipeError as e: - log.debug(f"BrokenPipeError in {fn.__qualname__}(): {e}") - except Exception as e: - log.error(f"Error in {fn.__qualname__}(): {e}") - log.trace(traceback.format_exc()) - except KeyboardInterrupt: - log.debug(f"Interrupted") - self.scan.stop() - except asyncio.CancelledError as e: - log.debug(f"{e}") - if callable(on_finish_callback): - try: - await self.scan.helpers.execute_sync_or_async(on_finish_callback) - except Exception as e: - log.error( - f"Error in on_finish_callback {on_finish_callback.__qualname__}() after {fn.__qualname__}(): {e}" - ) - log.trace(traceback.format_exc()) - return ret - async def _register_running(self, callback, *args, **kwargs): with self._task_counter: return await callback(*args, **kwargs) - def distribute_event(self, *args, **kwargs): + async def distribute_event(self, *args, **kwargs): """ Queue event with modules """ - event = self.scan.make_event(*args, **kwargs) + async with self.scan.acatch(context=self.distribute_event): + event = self.scan.make_event(*args, **kwargs) - event_hash = hash(event) - dup = event_hash in self.events_distributed - if dup: - self.scan.verbose(f"{event.module}: Duplicate event: {event}") - else: - self.events_distributed.add(event_hash) - # absorb event into the word cloud if it's in scope - if not dup and -1 < event.scope_distance < 1: - self.scan.word_cloud.absorb_event(event) - for mod in self.scan.modules.values(): - if not dup or mod.accept_dupes: - mod.queue_event(event) + event_hash = hash(event) + dup = event_hash in self.events_distributed + if dup: + self.scan.verbose(f"{event.module}: Duplicate event: {event}") + else: + self.events_distributed.add(event_hash) + # absorb event into the word cloud if it's in scope + if not dup and -1 < event.scope_distance < 1: + self.scan.word_cloud.absorb_event(event) + for mod in self.scan.modules.values(): + if not dup or mod.accept_dupes: + await mod.queue_event(event) async def _worker_loop(self): try: diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index df7bb80e15..f2f3fee371 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1,12 +1,14 @@ import queue + +# import signal import asyncio import logging import traceback +import contextlib from sys import exc_info from pathlib import Path from datetime import datetime from omegaconf import OmegaConf -from contextlib import suppress from collections import OrderedDict from bbot import config as bbot_config @@ -22,7 +24,7 @@ from bbot.core.logger import init_logging, get_log_level from bbot.core.helpers.names_generator import random_name from bbot.core.configurator.environ import prepare_environment -from bbot.core.errors import BBOTError, ScanError, ScanCancelledError, ValidationError +from bbot.core.errors import BBOTError, ScanError, ValidationError log = logging.getLogger("bbot.scanner") @@ -159,7 +161,15 @@ def __init__( self._finished_init = False self._cleanedup = False + self._loop = asyncio.get_event_loop() + + def _on_keyboard_interrupt(self, loop, event): + self.stop() + async def prep(self): + # event = asyncio.Event() + # self._loop.add_signal_handler(signal.SIGINT, self._on_keyboard_interrupt, loop, event) + self.helpers.mkdir(self.home) if not self._prepped: start_msg = f"Scan with {len(self._scan_modules):,} modules seeded with {len(self.target):,} targets" @@ -224,14 +234,7 @@ async def start(self): while 1: # abort if we're aborting if self.aborting: - # Empty event queues - for module in self.modules.values(): - with suppress(queue.Empty): - while 1: - module.incoming_event_queue.get_nowait() - with suppress(queue.Empty): - while 1: - self.incoming_event_queue.get_nowait() + self.drain_queues() break if "python" in self.modules: @@ -239,48 +242,37 @@ async def start(self): for e in events: yield e + # if initialization finished and the scan is no longer active if self._finished_init and not self.manager.active: - # And if new events were generated since last time we were here - if self.manager._new_activity: - self.manager._new_activity = False - self.status = "FINISHING" - # Trigger .finished() on every module and start over - log.info("Finishing scan") - finished_event = self.make_event("FINISHED", "FINISHED", dummy=True) - for module in self.modules.values(): - module.queue_event(finished_event) - else: - # Otherwise stop the scan if no new events were generated since last time + new_activity = await self.finish() + if not new_activity: break await asyncio.sleep(0.01) - # for module in self.modules.values(): - # for task in module.tasks: - # await task - - failed = False - - except KeyboardInterrupt: - self.stop() failed = False - except ScanCancelledError: - self.debug("Scan cancelled") - - except ScanError as e: - self.error(f"{e}") + except BaseException as e: + exception_chain = self.helpers.get_exception_chain(e) + if any(isinstance(exc, KeyboardInterrupt) for exc in exception_chain): + self.stop() + failed = False + else: + try: + raise + except ScanError as e: + self.error(f"{e}") - except BBOTError as e: - self.critical(f"Error during scan: {e}") - self.trace() + except BBOTError as e: + self.critical(f"Error during scan: {e}") + self.trace() - except Exception: - self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") + except Exception: + self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") finally: init_events_task.cancel() - with suppress(asyncio.CancelledError): + with contextlib.suppress(asyncio.CancelledError): await init_events_task await self.report() @@ -297,12 +289,12 @@ async def start(self): self.status = "FINISHED" ticker_task.cancel() - with suppress(asyncio.CancelledError): + with contextlib.suppress(asyncio.CancelledError): await ticker_task for t in manager_worker_loop_tasks: t.cancel() - with suppress(asyncio.CancelledError): + with contextlib.suppress(asyncio.CancelledError): await t scan_run_time = datetime.now() - scan_start_time @@ -346,16 +338,49 @@ async def setup_modules(self, remove_failed=True): elif total_failed > 0: self.warning(f"Setup failed for {total_failed:,} modules") - def stop(self, wait=False): + def stop(self): if self.status != "ABORTING": self.status = "ABORTING" self.hugewarning(f"Aborting scan") + self.trace() + self.drain_queues() self.helpers.kill_children() + self.drain_queues() self.helpers.kill_children() + async def finish(self): + # if new events were generated since last time we were here + if self.manager._new_activity: + self.manager._new_activity = False + self.status = "FINISHING" + # Trigger .finished() on every module and start over + log.info("Finishing scan") + finished_event = self.make_event("FINISHED", "FINISHED", dummy=True) + for module in self.modules.values(): + await module.queue_event(finished_event) + self.verbose("Completed finish()") + return True + # Return False if no new events were generated since last time + self.verbose("Completed final finish()") + return False + + def drain_queues(self): + # Empty event queues + self.debug("Draining queues") + for module in self.modules.values(): + with contextlib.suppress(asyncio.queues.QueueEmpty): + while 1: + module.incoming_event_queue.get_nowait() + with contextlib.suppress(queue.Empty): + while 1: + self.manager.incoming_event_queue.get_nowait() + self.debug("Finished draining queues") + async def report(self): for mod in self.modules.values(): - await self.manager.catch(mod._register_running, mod.report) + async with self.acatch(context=mod.report): + with mod._task_counter: + await mod.report() async def cleanup(self): # clean up modules @@ -364,7 +389,7 @@ async def cleanup(self): await mod._cleanup() if not self._cleanedup: self._cleanedup = True - with suppress(Exception): + with contextlib.suppress(Exception): self.home.rmdir() self.helpers.clean_old_scans() @@ -606,6 +631,53 @@ def _load_modules(self, modules): return loaded_modules, failed async def _status_ticker(self, interval=15): - while not self.stopped: - await asyncio.sleep(interval) - await self.manager.modules_status(_log=True) + async with self.acatch(): + # while not self.stopped: + while 1: + await asyncio.sleep(interval) + await self.manager.modules_status(_log=True) + + @contextlib.contextmanager + def catch(self, context="scan", finally_callback=None): + """ + Handle common errors by stopping scan, logging tracebacks, etc. + + with catch(): + do_stuff() + """ + try: + yield + except BaseException as e: + self._handle_exception(e, context=context) + + @contextlib.asynccontextmanager + async def acatch(self, context="scan", finally_callback=None): + """ + Async version of catch() + + async with catch(): + await do_stuff() + """ + try: + yield + except BaseException as e: + self._handle_exception(e, context=context) + + def _handle_exception(self, e, context="scan", finally_callback=None): + if callable(context): + context = f"{context.__qualname__}()" + filename, lineno, funcname = self.helpers.get_traceback_details(e) + exception_chain = self.helpers.get_exception_chain(e) + if any(isinstance(exc, KeyboardInterrupt) for exc in exception_chain): + log.debug(f"Interrupted") + self.stop() + elif isinstance(e, BrokenPipeError): + log.debug(f"BrokenPipeError in {filename}:{lineno}:{funcname}(): {e}") + elif isinstance(e, asyncio.CancelledError): + log.debug(f"asyncio CancelledError: {e}") + log.trace(traceback.format_exc()) + elif isinstance(e, Exception): + log.error(f"Error in {context}: {filename}:{lineno}:{funcname}(): {e}") + log.trace(traceback.format_exc()) + if callable(finally_callback): + self.helpers.execute_sync_or_async(finally_callback, e) diff --git a/bbot/test/test_step_2/test_command.py b/bbot/test/test_step_2/test_command.py index db2a069290..5f56d248b0 100644 --- a/bbot/test/test_step_2/test_command.py +++ b/bbot/test/test_step_2/test_command.py @@ -9,6 +9,8 @@ async def test_command(bbot_scanner, bbot_config): assert "plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"])).stdout result = (await scan1.helpers.run(["cat"], input="some\nrandom\nstdin")).stdout assert result.splitlines() == ["some", "random", "stdin"] + result = (await scan1.helpers.run(["cat"], input=["some", "random", "stdin"])).stdout + assert result.splitlines() == ["some", "random", "stdin"] # run_live lines = [] @@ -19,6 +21,10 @@ async def test_command(bbot_scanner, bbot_config): async for line in scan1.helpers.run_live(["cat"], input="some\nrandom\nstdin"): lines.append(line) assert lines == ["some", "random", "stdin"] + lines = [] + async for line in scan1.helpers.run_live(["cat"], input=["some", "random", "stdin"]): + lines.append(line) + assert lines == ["some", "random", "stdin"] # test piping lines = [] diff --git a/bbot/test/test_step_2/test_manager.py b/bbot/test/test_step_2/test_manager.py index 4a0bcff54a..eff5190e4d 100644 --- a/bbot/test/test_step_2/test_manager.py +++ b/bbot/test/test_step_2/test_manager.py @@ -89,23 +89,10 @@ class DummyModule3: # error catching msg = "Ignore this error, it belongs here" - - def raise_e(): - raise Exception(msg) - - def raise_k(): - raise KeyboardInterrupt(msg) - - def raise_s(): - raise ScanCancelledError(msg) - - def raise_b(): - raise BrokenPipeError(msg) - - manager.catch(raise_e, _on_finish_callback=raise_e) - manager.catch(raise_k) - manager.catch(raise_s) - manager.catch(raise_b) + exceptions = (Exception(msg), KeyboardInterrupt(msg), BrokenPipeError(msg)) + for e in exceptions: + with manager.catch(): + raise e def test_scope_distance(bbot_scanner, bbot_config): From 7e8b3c01624d6a8671d26948fd596f0e7b3d9487 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 10 May 2023 14:32:07 -0400 Subject: [PATCH 011/387] httpx tests --- bbot/core/helpers/dns.py | 10 ---------- bbot/core/helpers/helper.py | 4 ---- bbot/modules/output/python.py | 2 +- bbot/scanner/scanner.py | 2 +- bbot/test/helpers.py | 12 +++++++++--- bbot/test/modules_test_classes.py | 4 ++-- bbot/test/test_step_1/test_modules_full.py | 5 +++-- 7 files changed, 16 insertions(+), 23 deletions(-) diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 2e1bd093fd..23fd8601f8 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -127,8 +127,6 @@ async def _resolve_hostname(self, query, **kwargs): parent_hash = hash(f"{parent}:{rdtype}") dns_cache_hash = hash(f"{query}:{rdtype}") while tries_left > 0: - # if self.parent_helper.scan_stopping: - # break try: try: results = self._dns_cache[dns_cache_hash] @@ -176,8 +174,6 @@ async def _resolve_ip(self, query, **kwargs): errors = [] dns_cache_hash = hash(f"{query}:PTR") while tries_left > 0: - # if self.parent_helper.scan_stopping: - # break try: if dns_cache_hash in self._dns_cache: result = self._dns_cache[dns_cache_hash] @@ -586,9 +582,3 @@ def _get_dummy_module(self, name): dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") self._dummy_modules[name] = dummy_module return dummy_module - - def dns_warning(self, msg): - msg_hash = hash(msg) - if msg_hash not in self._dns_warnings: - log.warning(msg) - self._dns_warnings.add(msg_hash) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index f25578d30b..7d0a00d7d7 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -81,10 +81,6 @@ def scan(self): self._scan = Scanner() return self._scan - @property - def scan_stopping(self): - return getattr(self.scan, "stopping", False) - @property def in_tests(self): return os.environ.get("BBOT_TESTING", "") == "True" diff --git a/bbot/modules/output/python.py b/bbot/modules/output/python.py index d7ec521d96..3aebfeb527 100644 --- a/bbot/modules/output/python.py +++ b/bbot/modules/output/python.py @@ -5,5 +5,5 @@ class python(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output via Python API"} - def _worker(self): + async def _worker(self): pass diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index f2f3fee371..80aae3e62b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -238,7 +238,7 @@ async def start(self): break if "python" in self.modules: - events, finish, report = self.modules["python"].events_waiting + events, finish, report = await self.modules["python"].events_waiting() for e in events: yield e diff --git a/bbot/test/helpers.py b/bbot/test/helpers.py index 50ae08635b..4654c036ca 100644 --- a/bbot/test/helpers.py +++ b/bbot/test/helpers.py @@ -1,7 +1,10 @@ +import logging import requests_mock from abc import abstractmethod from omegaconf import OmegaConf +log = logging.getLogger("bbot.test.helpers") + class MockHelper: targets = ["blacklanternsecurity.com"] @@ -13,17 +16,16 @@ class MockHelper: def __init__(self, config, bbot_scanner, *args, **kwargs): self.name = kwargs.get("module_name", self.__class__.__name__.lower()) self.config = OmegaConf.merge(config, OmegaConf.create(self.config_overrides)) + modules = [self.name] + self.additional_modules self.scan = bbot_scanner( *self.targets, - modules=[self.name] + self.additional_modules, + modules=modules, name=f"{self.name}_test", config=self.config, whitelist=self.whitelist, blacklist=self.blacklist, ) self.patch_scan(self.scan) - self.scan.prep() - self.module = self.scan.modules[self.name] self.setup() def patch_scan(self, scan): @@ -41,6 +43,10 @@ def run(self): def check_events(self, events): raise NotImplementedError + @property + def module(self): + return self.scan.modules[self.name] + class RequestMockHelper(MockHelper): @abstractmethod diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index aedf31891a..c1bf5bf552 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -13,8 +13,8 @@ def mock_args(self): respond_args = dict(response_data=json.dumps({"foo": "bar"})) self.set_expect_requests(request_args, respond_args) - def run(self): - events = list(self.scan.start()) + async def run(self): + events = [e async for e in self.scan.start()] assert self.check_events(events) def check_events(self, events): diff --git a/bbot/test/test_step_1/test_modules_full.py b/bbot/test/test_step_1/test_modules_full.py index 8203f14acf..8d1f58204b 100644 --- a/bbot/test/test_step_1/test_modules_full.py +++ b/bbot/test/test_step_1/test_modules_full.py @@ -11,9 +11,10 @@ def test_gowitness(bbot_config, bbot_scanner, bbot_httpserver): x.run() -def test_httpx(bbot_config, bbot_scanner, bbot_httpserver): +@pytest.mark.asyncio +async def test_httpx(bbot_config, bbot_scanner, bbot_httpserver): x = Httpx(bbot_config, bbot_scanner, bbot_httpserver) - x.run() + await x.run() def test_excavate(bbot_config, bbot_scanner, bbot_httpserver): From 700347bafa270deee2a41f4b19ef546f0a767cd9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 11 May 2023 16:35:19 -0400 Subject: [PATCH 012/387] module tests up to masscan --- bbot/core/helpers/command.py | 39 ++- bbot/core/helpers/depsinstaller/installer.py | 29 +- bbot/core/helpers/diff.py | 143 +++++----- bbot/core/helpers/misc.py | 17 ++ bbot/modules/anubisdb.py | 8 +- bbot/modules/badsecrets.py | 2 +- bbot/modules/base.py | 91 ++++--- bbot/modules/crobat.py | 14 +- bbot/modules/fingerprintx.py | 4 +- bbot/modules/gowitness.py | 25 +- bbot/modules/leakix.py | 4 +- bbot/modules/masscan.py | 15 +- bbot/modules/paramminer_cookies.py | 4 +- bbot/modules/paramminer_getparams.py | 4 +- bbot/modules/paramminer_headers.py | 29 +- bbot/modules/robots.py | 6 +- bbot/modules/secretsdb.py | 6 +- bbot/modules/subdomain_hijack.py | 22 +- bbot/modules/telerik.py | 54 ++-- bbot/scanner/manager.py | 14 +- bbot/scanner/scanner.py | 49 ++-- bbot/test/helpers.py | 49 ++-- bbot/test/modules_test_classes.py | 251 +++++++++--------- bbot/test/test_step_1/test_modules_full.py | 263 +++++++++++-------- bbot/test/test_step_2/test_command.py | 38 ++- poetry.lock | 41 ++- pyproject.toml | 2 +- 27 files changed, 689 insertions(+), 534 deletions(-) diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 219fa1bc73..be66d04eec 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -2,14 +2,14 @@ import asyncio import logging import traceback -from subprocess import CompletedProcess +from subprocess import CompletedProcess, CalledProcessError from .misc import smart_decode, smart_encode log = logging.getLogger("bbot.core.helpers.command") -async def run(self, *command, **kwargs): +async def run(self, *command, check=False, text=True, **kwargs): """ Simple helper for running a command, and getting its output as a string process = await run(["ls", "/tmp"]) @@ -25,16 +25,20 @@ async def run(self, *command, **kwargs): stdout, stderr = await proc.communicate(_input) # surface stderr - stderr = smart_decode(stderr) - stdout = smart_decode(stdout) - if stderr and proc.returncode != 0: - command_str = " ".join(command) - log.warning(f"Stderr for run({command_str}):\n\t{stderr}") + if text: + stderr = smart_decode(stderr) + stdout = smart_decode(stdout) + if proc.returncode: + if check: + raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr) + if stderr: + command_str = " ".join(command) + log.warning(f"Stderr for run({command_str}):\n\t{stderr}") return CompletedProcess(command, proc.returncode, stdout, stderr) -async def run_live(self, *command, **kwargs): +async def run_live(self, *command, check=False, text=True, **kwargs): """ Simple helper for running a command and iterating through its output line by line in realtime async for line in run_live(["ls", "/tmp"]): @@ -50,7 +54,11 @@ async def run_live(self, *command, **kwargs): line = await proc.stdout.readline() if not line: break - yield smart_decode(line).rstrip("\r\n") + if text: + line = smart_decode(line).rstrip("\r\n") + else: + line = line.rstrip(b"\r\n") + yield line if input_task is not None: try: @@ -59,12 +67,17 @@ async def run_live(self, *command, **kwargs): log.trace(traceback.format_exc()) await proc.wait() - # surface stderr - if proc.returncode != 0: + if proc.returncode: stdout, stderr = await proc.communicate() + if text: + stderr = smart_decode(stderr) + stdout = smart_decode(stdout) + if check: + raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr) + # surface stderr if stderr: command_str = " ".join(command) - log.warning(f"Stderr for run_live({command_str}):\n\t{smart_decode(stderr)}") + log.warning(f"Stderr for run_live({command_str}):\n\t{stderr}") async def _spawn_proc(self, *command, **kwargs): @@ -100,6 +113,8 @@ async def _write_stdin(proc, _input): def _prepare_command_kwargs(self, command, kwargs): + # limit = 10MB (this is needed for cases like httpx that are sending large JSON blobs over stdout) + kwargs["limit"] = 1024 * 1024 * 10 if not "stdout" in kwargs: kwargs["stdout"] = asyncio.subprocess.PIPE if not "stderr" in kwargs: diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 31c37a6dba..2195237f38 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -29,6 +29,7 @@ def __init__(self, parent_helper): os.environ["ANSIBLE_TIMEOUT"] = str(http_timeout) self.askpass_filename = "sudo_askpass.py" + self._installed_sudo_askpass = False self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None) if self._sudo_password is None: if configurator.bbot_sudo_pass is not None: @@ -55,7 +56,7 @@ def __init__(self, parent_helper): self.ensure_root_lock = Lock() - def install(self, *modules): + async def install(self, *modules): self.install_core_deps() succeeded = [] failed = [] @@ -95,7 +96,7 @@ def install(self, *modules): # get sudo access if we need it if preloaded.get("sudo", False) == True: self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.') - success = self.install_module(m) + success = await self.install_module(m) self.setup_status[module_hash] = success if success or self.ignore_failed_deps: log.debug(f'Setup succeeded for module "{m}"') @@ -122,7 +123,7 @@ def install(self, *modules): failed.sort() return succeeded, failed - def install_module(self, module): + async def install_module(self, module): success = True preloaded = self.all_modules_preloaded[module] @@ -145,11 +146,11 @@ def install_module(self, module): deps_pip = preloaded["deps"]["pip"] deps_pip_constraints = preloaded["deps"]["pip_constraints"] if deps_pip: - success &= self.pip_install(deps_pip, constraints=deps_pip_constraints) + success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints) return success - def pip_install(self, packages, constraints=None): + async def pip_install(self, packages, constraints=None): packages_str = ",".join(packages) log.info(f"Installing the following pip packages: {packages_str}") @@ -162,7 +163,7 @@ def pip_install(self, packages, constraints=None): process = None try: - process = self.parent_helper.run(command, check=True) + process = await self.parent_helper.run(command, check=True) message = f'Successfully installed pip packages "{packages_str}"' output = process.stdout.splitlines()[-1] if output: @@ -297,6 +298,7 @@ def write_setup_status(self): json.dump(self.setup_status, f) def ensure_root(self, message=""): + self._install_sudo_askpass() with self.ensure_root_lock: if os.geteuid() != 0 and self._sudo_password is None: if message: @@ -314,11 +316,7 @@ def ensure_root(self, message=""): def install_core_deps(self): to_install = set() - # install custom askpass script - askpass_src = Path(__file__).resolve().parent / self.askpass_filename - askpass_dst = self.parent_helper.tools_dir / self.askpass_filename - shutil.copy(askpass_src, askpass_dst) - askpass_dst.chmod(askpass_dst.stat().st_mode | stat.S_IEXEC) + self._install_sudo_askpass() # ensure tldextract data is cached self.parent_helper.tldextract("evilcorp.co.uk") # command: package_name @@ -329,3 +327,12 @@ def install_core_deps(self): if to_install: self.ensure_root() self.apt_install(list(to_install)) + + def _install_sudo_askpass(self): + if not self._installed_sudo_askpass: + self._installed_sudo_askpass = True + # install custom askpass script + askpass_src = Path(__file__).resolve().parent / self.askpass_filename + askpass_dst = self.parent_helper.tools_dir / self.askpass_filename + shutil.copy(askpass_src, askpass_dst) + askpass_dst.chmod(askpass_dst.stat().st_mode | stat.S_IEXEC) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 43f668dfd8..2f191e0db6 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -1,6 +1,5 @@ import logging import xmltodict -from time import sleep from deepdiff import DeepDiff from contextlib import suppress from xml.parsers.expat import ExpatError @@ -14,69 +13,77 @@ def __init__(self, baseline_url, parent_helper, method="GET", allow_redirects=Fa self.parent_helper = parent_helper self.baseline_url = baseline_url self.include_cache_buster = include_cache_buster + self.method = method + self.allow_redirects = allow_redirects + self._baselined = False + + async def _baseline(self): + if not self._baselined: + self._baselined = True + # vanilla URL + if self.include_cache_buster: + url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl() + else: + url_1 = self.baseline_url + baseline_1 = await self.parent_helper.request( + url_1, follow_redirects=self.allow_redirects, method=self.method + ) + await self.parent_helper.sleep(1) + # put random parameters in URL, headers, and cookies + get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)} + + if self.include_cache_buster: + get_params.update(self.gen_cache_buster()) + url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl() + baseline_2 = await self.parent_helper.request( + url_2, + headers={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, + cookies={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, + follow_redirects=self.allow_redirects, + method=self.method, + ) - # vanilla URL - if self.include_cache_buster: - url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl() - else: - url_1 = self.baseline_url - baseline_1 = self.parent_helper.request(url_1, allow_redirects=allow_redirects, method=method) - sleep(1) - # put random parameters in URL, headers, and cookies - get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)} - - if self.include_cache_buster: - get_params.update(self.gen_cache_buster()) - url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl() - baseline_2 = self.parent_helper.request( - url_2, - headers={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, - cookies={self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, - allow_redirects=allow_redirects, - method=method, - ) - - self.baseline = baseline_1 - - if baseline_1 is None or baseline_2 is None: - log.debug("HTTP error while establishing baseline, aborting") - raise HttpCompareError("Can't get baseline from source URL") - if baseline_1.status_code != baseline_2.status_code: - log.debug("Status code not stable during baseline, aborting") - raise HttpCompareError("Can't get baseline from source URL") - try: - baseline_1_json = xmltodict.parse(baseline_1.text) - baseline_2_json = xmltodict.parse(baseline_2.text) - except ExpatError: - log.debug(f"Cant HTML parse for {baseline_url}. Switching to text parsing as a backup") - baseline_1_json = baseline_1.text.split("\n") - baseline_2_json = baseline_2.text.split("\n") - - ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree") - self.ddiff_filters = [] - - for k, v in ddiff.items(): - for x in list(ddiff[k]): - log.debug(f"Added {k} filter for path: {x.path()}") - self.ddiff_filters.append(x.path()) - - self.baseline_json = baseline_1_json - - self.baseline_ignore_headers = [ - h.lower() - for h in [ - "date", - "last-modified", - "content-length", - "ETag", - "X-Pad", - "X-Backside-Transport", + self.baseline = baseline_1 + + if baseline_1 is None or baseline_2 is None: + log.debug("HTTP error while establishing baseline, aborting") + raise HttpCompareError("Can't get baseline from source URL") + if baseline_1.status_code != baseline_2.status_code: + log.debug("Status code not stable during baseline, aborting") + raise HttpCompareError("Can't get baseline from source URL") + try: + baseline_1_json = xmltodict.parse(baseline_1.text) + baseline_2_json = xmltodict.parse(baseline_2.text) + except ExpatError: + log.debug(f"Cant HTML parse for {self.baseline_url}. Switching to text parsing as a backup") + baseline_1_json = baseline_1.text.split("\n") + baseline_2_json = baseline_2.text.split("\n") + + ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree") + self.ddiff_filters = [] + + for k, v in ddiff.items(): + for x in list(ddiff[k]): + log.debug(f"Added {k} filter for path: {x.path()}") + self.ddiff_filters.append(x.path()) + + self.baseline_json = baseline_1_json + + self.baseline_ignore_headers = [ + h.lower() + for h in [ + "date", + "last-modified", + "content-length", + "ETag", + "X-Pad", + "X-Backside-Transport", + ] ] - ] - dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers) + dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers) - self.baseline_ignore_headers += [x.lower() for x in dynamic_headers] - self.baseline_body_distance = self.compare_body(baseline_1_json, baseline_2_json) + self.baseline_ignore_headers += [x.lower() for x in dynamic_headers] + self.baseline_body_distance = self.compare_body(baseline_1_json, baseline_2_json) def gen_cache_buster(self): return {self.parent_helper.rand_string(6): "1"} @@ -114,7 +121,7 @@ def compare_body(self, content_1, content_2): log.debug(ddiff) return False - def compare( + async def compare( self, subject, headers=None, cookies=None, check_reflection=False, method="GET", allow_redirects=False ): """ @@ -125,6 +132,7 @@ def compare( "reason" is the location of the change ("code", "body", "header", or None), and "reflection" is whether the value was reflected in the HTTP response """ + await self._baseline() reflection = False if self.include_cache_buster: @@ -132,8 +140,8 @@ def compare( url = self.parent_helper.add_get_params(subject, {cache_key: cache_value}).geturl() else: url = subject - subject_response = self.parent_helper.request( - url, headers=headers, cookies=cookies, allow_redirects=allow_redirects, method=method + subject_response = await self.parent_helper.request( + url, headers=headers, cookies=cookies, follow_redirects=allow_redirects, method=method ) if not subject_response: @@ -184,10 +192,11 @@ def compare( else: return (False, diff_reasons, reflection, subject_response) - def canary_check(self, url, mode, rounds=6): + async def canary_check(self, url, mode, rounds=6): """ test detection using a canary to find hosts giving bad results """ + await self._baseline() headers = None cookies = None for i in range(0, rounds): @@ -202,7 +211,9 @@ def canary_check(self, url, mode, rounds=6): else: raise ValueError(f'Invalid mode: "{mode}", choose from: getparam, header, cookie') - match, reasons, reflection, subject_response = self.compare(new_url, headers=headers, cookies=cookies) + match, reasons, reflection, subject_response = await self.compare( + new_url, headers=headers, cookies=cookies + ) # a nonsense header "caused" a difference, we need to abort if match == False: diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 5e95899200..243901231f 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -10,6 +10,7 @@ import shutil import signal import string +import asyncio import difflib import inspect import logging @@ -18,6 +19,7 @@ import traceback import subprocess as sp from pathlib import Path +from asyncio import sleep # noqa from itertools import islice from datetime import datetime from tabulate import tabulate @@ -1052,3 +1054,18 @@ def get_traceback_details(e): lineno = last_frame.lineno funcname = last_frame.name return filename, lineno, funcname + + +def create_task(*args, **kwargs): + return asyncio.create_task(*args, **kwargs) + + +def as_completed(*args, **kwargs): + yield from asyncio.as_completed(*args, **kwargs) + + +async def cancel_tasks(tasks): + for task in tasks: + task.cancel() + with suppress(asyncio.CancelledError): + await task diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index affb579eec..c580c9e9da 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -10,9 +10,9 @@ class anubisdb(crobat): base_url = "https://jldc.me/anubis/subdomains" dns_abort_depth = 5 - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) def abort_if_pre(self, hostname): """ @@ -24,11 +24,11 @@ def abort_if_pre(self, hostname): return True return False - def abort_if(self, event): + async def abort_if(self, event): # abort if dns name is unresolved if not "resolved" in event.tags: return True, "DNS name is unresolved" - return super().abort_if(event) + return await super().abort_if(event) def parse_results(self, r, query): results = set() diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 82bf3c1453..99194eae3d 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -11,7 +11,7 @@ class badsecrets(BaseModule): max_event_handlers = 2 deps_pip = ["badsecrets~=0.1.287"] - def handle_event(self, event): + async def handle_event(self, event): resp_body = event.data.get("body", None) resp_headers = event.data.get("header", None) resp_cookies = {} diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 75c9f08c40..4d1f626541 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1,4 +1,3 @@ -import queue import asyncio import logging import traceback @@ -98,6 +97,7 @@ def __init__(self, scan): # track number of failures (for .request_with_fail_count()) self._request_failures = 0 + self._tasks = [] self._event_received = asyncio.Condition() async def setup(self): @@ -238,10 +238,13 @@ def make_event(self, *args, **kwargs): def emit_event(self, *args, **kwargs): event_kwargs = dict(kwargs) + emit_kwargs = {} for o in ("on_success_callback", "abort_if", "quick"): - event_kwargs.pop(o, None) + v = event_kwargs.pop(o, None) + if v is not None: + emit_kwargs[o] = v event = self.make_event(*args, **event_kwargs) - self.scan.manager.queue_event(event) + self.scan.manager.queue_event(event, **emit_kwargs) async def events_waiting(self): """ @@ -262,6 +265,7 @@ async def events_waiting(self): finish = True else: events.append(event) + self.scan.stats.event_consumed(event, self) elif reason: self.debug(f"Not accepting {event} because {reason}") except asyncio.queues.QueueEmpty: @@ -276,7 +280,7 @@ def num_queued_events(self): return ret def start(self): - self.tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] + self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] async def _setup(self): status_codes = {False: "hard-fail", None: "soft-fail", True: "success"} @@ -300,40 +304,42 @@ async def _setup(self): return self.name, status, str(msg) async def _worker(self): - while not self.scan.stopping: - # hold the reigns if our outgoing queue is full - # if self._qsize and self.outgoing_event_queue.qsize() >= self._qsize: - # with self.event_received: - # await self.event_received.wait() - - if self.batch_size > 1: - submitted = await self._handle_batch() - if not submitted: - async with self._event_received: - await self._event_received.wait() - - else: - try: - if self.incoming_event_queue: - event = await self.incoming_event_queue.get() - else: - self.debug(f"Event queue is in bad state") - return - except asyncio.queues.QueueEmpty: - continue - self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") - acceptable, reason = await self._event_postcheck(event) - if not acceptable: - self.debug(f"Not accepting {event} because {reason}") - if acceptable: - if event.type == "FINISHED": - async with self.scan.acatch(context=f"{self.name}.finish"): - with self._task_counter: - await self.finish() - else: - async with self.scan.acatch(context=f"{self.name}.handle_event"): - with self._task_counter: - await self.handle_event(event) + async with self.scan.acatch(context=self._worker): + while not self.scan.stopping: + # hold the reigns if our outgoing queue is full + # if self._qsize and self.outgoing_event_queue.qsize() >= self._qsize: + # with self.event_received: + # await self.event_received.wait() + + if self.batch_size > 1: + submitted = await self._handle_batch() + if not submitted: + async with self._event_received: + await self._event_received.wait() + + else: + try: + if self.incoming_event_queue: + event = await self.incoming_event_queue.get() + else: + self.debug(f"Event queue is in bad state") + return + except asyncio.queues.QueueEmpty: + continue + self.debug(f"Got {event} from {getattr(event, 'module', 'unknown_module')}") + acceptable, reason = await self._event_postcheck(event) + if not acceptable: + self.debug(f"Not accepting {event} because {reason}") + if acceptable: + if event.type == "FINISHED": + async with self.scan.acatch(context=f"{self.name}.finish"): + with self._task_counter: + await self.finish() + else: + self.scan.stats.event_consumed(event, self) + async with self.scan.acatch(context=f"{self.name}.handle_event"): + with self._task_counter: + await self.handle_event(event) @property def max_scope_distance(self): @@ -454,7 +460,7 @@ def set_error_state(self, message=None): # clear incoming queue if self.incoming_event_queue: self.debug(f"Emptying event_queue") - with suppress(queue.Empty): + with suppress(asyncio.queues.QueueEmpty): while 1: self.incoming_event_queue.get_nowait() # set queue to None to prevent its use @@ -493,8 +499,8 @@ def finished(self): """ return not self.running and self.num_queued_events <= 0 and self.outgoing_event_queue.qsize() <= 0 - def request_with_fail_count(self, *args, **kwargs): - r = self.helpers.request(*args, **kwargs) + async def request_with_fail_count(self, *args, **kwargs): + r = await self.helpers.request(*args, **kwargs) if r is None: self._request_failures += 1 else: @@ -550,8 +556,7 @@ def memory_usage(self): """ Return how much memory the module is currently using in bytes """ - seen = set(self.scan.pools.values()) - seen.update({self.scan, self.helpers, self.log}) + seen = {self.scan, self.helpers, self.log} return get_size(self, max_depth=3, seen=seen) def __str__(self): diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py index 66c0f70b62..da4191911f 100644 --- a/bbot/modules/crobat.py +++ b/bbot/modules/crobat.py @@ -77,17 +77,17 @@ def already_processed(self, hostname): return True return False - def abort_if(self, event): + async def abort_if(self, event): # this helps weed out unwanted results when scanning IP_RANGES and wildcard domains if "in-scope" not in event.tags: return True - if self._is_wildcard(event.data): + if await self._is_wildcard(event.data): return True return False async def handle_event(self, event): query = self.make_query(event) - results = self.query(query) + results = await self.query(query) if results: for hostname in set(results): if hostname: @@ -99,9 +99,9 @@ async def handle_event(self, event): if hostname and hostname.endswith(f".{query}") and not hostname == event.data: self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) def make_query(self, event): if "target" in event.tags: @@ -116,13 +116,13 @@ def parse_results(self, r, query=None): for hostname in json: yield hostname - def query(self, query, parse_fn=None, request_fn=None): + async def query(self, query, parse_fn=None, request_fn=None): if parse_fn is None: parse_fn = self.parse_results if request_fn is None: request_fn = self.request_url try: - results = list(parse_fn(request_fn(query), query)) + results = list(parse_fn(await request_fn(query), query)) if results: return results self.debug(f'No results for "{query}"') diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index e6b76227e0..3bc36da66f 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -26,10 +26,10 @@ class fingerprintx(BaseModule): }, ] - def handle_batch(self, *events): + async def handle_batch(self, *events): _input = {e.data: e for e in events} command = ["fingerprintx", "--json"] - for line in self.helpers.run_live(command, input=list(_input), stderr=subprocess.DEVNULL): + async for line in self.helpers.run_live(command, input=list(_input), stderr=subprocess.DEVNULL): try: j = json.loads(line) except Exception as e: diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 3972f4f105..867f1534bc 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -80,7 +80,7 @@ class gowitness(BaseModule): # this is one hop further than the default scope_distance_modifier = 1 - def setup(self): + async def setup(self): self.timeout = self.config.get("timeout", 10) self.threads = self.config.get("threads", 4) self.proxy = self.scan.config.get("http_proxy", "") @@ -113,7 +113,7 @@ def prep(self): copymode(self.helpers.tools_dir / "gowitness", self.base_path / "gowitness") self.prepped = True - def filter_event(self, event): + async def filter_event(self, event): # Ignore URLs that are redirects if any(t.startswith("status-30") for t in event.tags): return False, "URL is a redirect" @@ -122,12 +122,12 @@ def filter_event(self, event): return False, "event is from self" return True - def handle_batch(self, *events): + async def handle_batch(self, *events): self.prep() stdin = "\n".join([str(e.data) for e in events]) events = {e.data: e for e in events} - for line in self.helpers.run_live(self.command, input=stdin): + async for line in self.helpers.run_live(self.command, input=stdin): self.debug(line) # emit web screenshots @@ -242,12 +242,11 @@ def cur_execute(self, cur, query): self.trace() return [] - def report(self): - with self._report_lock: - if self.screenshots_taken: - self.success(f"{len(self.screenshots_taken):,} web screenshots captured. To view:") - self.success(f" - Start gowitness") - self.success(f" - cd {self.base_path} && ./gowitness server") - self.success(f" - Browse to http://localhost:7171") - else: - self.info(f"No web screenshots captured") + async def report(self): + if self.screenshots_taken: + self.success(f"{len(self.screenshots_taken):,} web screenshots captured. To view:") + self.success(f" - Start gowitness") + self.success(f" - cd {self.base_path} && ./gowitness server") + self.success(f" - Browse to http://localhost:7171") + else: + self.info(f"No web screenshots captured") diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index e692ae838f..d494683afc 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -9,9 +9,9 @@ class leakix(crobat): base_url = "https://leakix.net" - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" - return self.request_with_fail_count(url, headers={"Accept": "application/json"}) + return await self.request_with_fail_count(url, headers={"Accept": "application/json"}) def parse_results(self, r, query=None): json = r.json() diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index b7f594f3e7..9e8282702f 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -50,7 +50,7 @@ class masscan(BaseModule): ] _qsize = 100 - def setup(self): + async def setup(self): self.ports = self.config.get("ports", "80,443") self.rate = self.config.get("rate", 600) self.wait = self.config.get("wait", 10) @@ -61,7 +61,7 @@ def setup(self): if not self.helpers.in_tests: try: dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True) - dry_run_result = self.helpers.run(dry_run_command) + dry_run_result = await self.helpers.run(dry_run_command) self.masscan_config = dry_run_result.stdout self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l) except subprocess.CalledProcessError as e: @@ -91,7 +91,7 @@ def setup(self): self.syn_cache_fd = None return True - def handle_event(self, event): + async def handle_event(self, event): if self.use_cache: self.emit_from_cache() else: @@ -114,7 +114,7 @@ def handle_event(self, event): if self.ping_first: self.verbose("Starting masscan (ping scan)") - self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True) + await self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True) targets = ",".join(str(h) for h in self.alive_hosts) if not targets: self.warning("No hosts responded to pings") @@ -123,13 +123,13 @@ def handle_event(self, event): # TCP SYN scan if self.ports: self.verbose("Starting masscan (TCP SYN scan)") - self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude) + await self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude) else: self.verbose("No ports specified, skipping TCP SYN scan") # save memory self.alive_hosts.clear() - def masscan(self, targets, result_callback, exclude=None, ping=False): + async def masscan(self, targets, result_callback, exclude=None, ping=False): # config file masscan_config = self.masscan_config.replace(self._target_findkey, targets) self.debug("Masscan config:") @@ -145,7 +145,8 @@ def masscan(self, targets, result_callback, exclude=None, ping=False): stats_file = self.helpers.tempfile_tail(callback=self.verbose) try: with open(stats_file, "w") as stats_fh: - for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh): + self.critical(f"masscan: {command}") + async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh): self.process_output(line, result_callback=result_callback) finally: stats_file.unlink() diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index 64f3965191..0d4e28b0e6 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -20,9 +20,9 @@ class paramminer_cookies(paramminer_headers): in_scope_only = True compare_mode = "cookie" - def check_batch(self, compare_helper, url, cookie_list): + async def check_batch(self, compare_helper, url, cookie_list): cookies = {p: self.rand_string(14) for p in cookie_list} - return compare_helper.compare(url, cookies=cookies) + return await compare_helper.compare(url, cookies=cookies) def gen_count_args(self, url): cookie_count = 40 diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 7ad778c0e4..6b79053434 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -19,9 +19,9 @@ class paramminer_getparams(paramminer_headers): in_scope_only = True compare_mode = "getparam" - def check_batch(self, compare_helper, url, getparam_list): + async def check_batch(self, compare_helper, url, getparam_list): test_getparams = {p: self.rand_string(14) for p in getparam_list} - return compare_helper.compare(self.helpers.add_get_params(url, test_getparams).geturl()) + return await compare_helper.compare(self.helpers.add_get_params(url, test_getparams).geturl()) def gen_count_args(self, url): getparam_count = 40 diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 1009a4a33f..8357ea4db9 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -29,28 +29,28 @@ class paramminer_headers(BaseModule): in_scope_only = True compare_mode = "header" - def setup(self): + async def setup(self): wordlist_url = self.config.get("wordlist", "") - self.wordlist = self.helpers.wordlist(wordlist_url) + self.wordlist = await self.helpers.wordlist(wordlist_url) return True def rand_string(self, *args, **kwargs): return self.helpers.rand_string(*args, **kwargs) - def handle_event(self, event): + async def handle_event(self, event): url = event.data try: compare_helper = self.helpers.http_compare(url) except HttpCompareError as e: self.debug(e) return - batch_size = self.count_test(url) + batch_size = await self.count_test(url) if batch_size == None or batch_size <= 0: self.debug(f"Failed to get baseline max {self.compare_mode} count, aborting") return self.debug(f"Resolved batch_size at {str(batch_size)}") - if compare_helper.canary_check(url, mode=self.compare_mode) == False: + if await compare_helper.canary_check(url, mode=self.compare_mode) == False: self.verbose(f'Aborting "{url}" due to failed canary check') return @@ -62,7 +62,7 @@ def handle_event(self, event): abort_threshold = 25 try: for group in self.helpers.grouper(wordlist_cleaned, batch_size): - for result, reasons, reflection in self.binary_search(compare_helper, url, group): + async for result, reasons, reflection in self.binary_search(compare_helper, url, group): results.add((result, ",".join(reasons), reflection)) if len(results) >= abort_threshold: self.warning( @@ -85,14 +85,14 @@ def handle_event(self, event): tags=tags, ) - def count_test(self, url): - baseline = self.helpers.request(url) + async def count_test(self, url): + baseline = await self.helpers.request(url) if baseline is None: return if str(baseline.status_code)[0] in ("4", "5"): return for count, args, kwargs in self.gen_count_args(url): - r = self.helpers.request(*args, **kwargs) + r = await self.helpers.request(*args, **kwargs) if r is not None and not ((str(r.status_code)[0] in ("4", "5"))): return count @@ -112,7 +112,7 @@ def clean_list(self, header): return True return False - def binary_search(self, compare_helper, url, group, reasons=None, reflection=False): + async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False): if reasons is None: reasons = [] self.debug(f"Entering recursive binary_search with {len(group):,} sized group") @@ -121,15 +121,16 @@ def binary_search(self, compare_helper, url, group, reasons=None, reflection=Fal yield group[0], reasons, reflection elif len(group) > 1: for group_slice in self.helpers.split_list(group): - match, reasons, reflection, subject_response = self.check_batch(compare_helper, url, group_slice) + match, reasons, reflection, subject_response = await self.check_batch(compare_helper, url, group_slice) if match == False: - yield from self.binary_search(compare_helper, url, group_slice, reasons, reflection) + async for r in self.binary_search(compare_helper, url, group_slice, reasons, reflection): + yield r else: self.warning(f"Submitted group of size 0 to binary_search()") - def check_batch(self, compare_helper, url, header_list): + async def check_batch(self, compare_helper, url, header_list): rand = self.rand_string() test_headers = {} for header in header_list: test_headers[header] = rand - return compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1)) + return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1)) diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index 97cdfffacf..da4908fcef 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -16,11 +16,11 @@ class robots(BaseModule): in_scope_only = True - def setup(self): + async def setup(self): self.scanned_hosts = set() return True - def handle_event(self, event): + async def handle_event(self, event): parsed_host = event.parsed host = f"{parsed_host.scheme}://{parsed_host.netloc}/" host_hash = hash(host) @@ -32,7 +32,7 @@ def handle_event(self, event): result = None url = f"{host}robots.txt" - result = self.helpers.request(url) + result = await self.helpers.request(url) if result: body = result.text diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py index 42ed0445f3..db74f7acc3 100644 --- a/bbot/modules/secretsdb.py +++ b/bbot/modules/secretsdb.py @@ -19,10 +19,10 @@ class secretsdb(BaseModule): } deps_pip = ["pyyaml~=6.0"] - def setup(self): + async def setup(self): self.rules = [] self.min_confidence = self.config.get("min_confidence", 99) - self.sig_file = self.helpers.wordlist(self.config.get("signatures", "")) + self.sig_file = await self.helpers.wordlist(self.config.get("signatures", "")) with open(self.sig_file) as f: rules_yaml = yaml.safe_load(f).get("patterns", []) for r in rules_yaml: @@ -41,7 +41,7 @@ def setup(self): self.debug(f"Error compiling regex: r'{regex}'") return True - def handle_event(self, event): + async def handle_event(self, event): resp_body = event.data.get("body", "") resp_headers = event.data.get("raw_header", "") for r in self.rules: diff --git a/bbot/modules/subdomain_hijack.py b/bbot/modules/subdomain_hijack.py index cfcc441139..201e8b4a36 100644 --- a/bbot/modules/subdomain_hijack.py +++ b/bbot/modules/subdomain_hijack.py @@ -1,6 +1,6 @@ import re import json -import requests +import httpx from bbot.modules.base import BaseModule from bbot.core.helpers.misc import tldextract @@ -18,9 +18,9 @@ class subdomain_hijack(BaseModule): scope_distance_modifier = 3 max_event_handlers = 5 - def setup(self): + async def setup(self): fingerprints_url = self.config.get("fingerprints") - fingerprints_file = self.helpers.wordlist(fingerprints_url) + fingerprints_file = await self.helpers.wordlist(fingerprints_url) with open(fingerprints_file) as f: fingerprints = json.load(f) self.fingerprints = [] @@ -40,8 +40,8 @@ def setup(self): self.debug(f"Successfully processed {len(self.fingerprints):,} fingerprints") return True - def handle_event(self, event): - hijackable, reason = self.check_subdomain(event) + async def handle_event(self, event): + hijackable, reason = await self.check_subdomain(event) if hijackable: source_hosts = [] e = event @@ -66,7 +66,7 @@ def handle_event(self, event): else: self.debug(reason) - def check_subdomain(self, event): + async def check_subdomain(self, event): for f in self.fingerprints: for domain in f.domains: self_matches = self.helpers.host_in_host(event.data, domain) @@ -81,25 +81,25 @@ def check_subdomain(self, event): return False, "Scan cancelled" # first, try base request url = f"{scheme}://{event.data}" - match, reason = self._verify_fingerprint(f, url, cache_for=60 * 60 * 24) + match, reason = await self._verify_fingerprint(f, url, cache_for=60 * 60 * 24) if match: return match, reason # next, try subdomain -[CNAME]-> other_domain url = f"{scheme}://{domain}" headers = {"Host": event.data} - match, reason = self._verify_fingerprint(f, url, headers=headers) + match, reason = await self._verify_fingerprint(f, url, headers=headers) if match: return match, reason return False, f'Subdomain "{event.data}" not hijackable' - def _verify_fingerprint(self, fingerprint, *args, **kwargs): + async def _verify_fingerprint(self, fingerprint, *args, **kwargs): kwargs["raise_error"] = True kwargs["timeout"] = 10 kwargs["retries"] = 0 if fingerprint.http_status is not None: kwargs["allow_redirects"] = False try: - r = self.helpers.request(*args, **kwargs) + r = await self.helpers.request(*args, **kwargs) if fingerprint.http_status is not None and r.status_code == fingerprint.http_status: return True, f"HTTP status == {fingerprint.http_status}" text = getattr(r, "text", "") @@ -109,7 +109,7 @@ def _verify_fingerprint(self, fingerprint, *args, **kwargs): and fingerprint.fingerprint_regex.findall(text) ): return True, "Fingerprint match" - except requests.exceptions.RequestException as e: + except httpx.RequestError as e: if fingerprint.nxdomain and "Name or service not known" in str(e): return True, f"NXDOMAIN" return False, "No match" diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 2465ae875e..5166e45b73 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -1,6 +1,8 @@ -from bbot.modules.base import BaseModule -from urllib.parse import urlparse +import asyncio from sys import executable +from urllib.parse import urlparse + +from bbot.modules.base import BaseModule class telerik(BaseModule): @@ -156,12 +158,12 @@ class telerik(BaseModule): max_event_handlers = 5 - def setup(self): + async def setup(self): self.scanned_hosts = set() self.timeout = self.scan.config.get("httpx_timeout", 5) return True - def handle_event(self, event): + async def handle_event(self, event): host = f"{event.parsed.scheme}://{event.parsed.netloc}/" host_hash = hash(host) if host_hash in self.scanned_hosts: @@ -171,7 +173,7 @@ def handle_event(self, event): self.scanned_hosts.add(host_hash) webresource = "Telerik.Web.UI.WebResource.axd?type=rau" - result = self.test_detector(event.data, webresource) + result, _ = await self.test_detector(event.data, webresource) if result: if "RadAsyncUpload handler is registered succesfully" in result.text: self.debug(f"Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)") @@ -198,7 +200,7 @@ def handle_event(self, event): str(root_tool_path / "testfile.txt"), result.url, ] - output = self.helpers.run(command) + output = await self.helpers.run(command) description = f"[CVE-2017-11317] [{str(version)}] {webresource}" if "fileInfo" in output.stdout: self.debug(f"Confirmed Vulnerable Telerik (version: {str(version)}") @@ -214,15 +216,16 @@ def handle_event(self, event): ) break - futures = {} + tasks = [] for dh in self.DialogHandlerUrls: - future = self.submit_task(self.test_detector, event.data, f"{dh}?dp=1") - futures[future] = dh + tasks.append(self.helpers.create_task(self.test_detector(event.data, f"{dh}?dp=1"))) fail_count = 0 - for future in self.helpers.as_completed(futures): - dh = futures[future] - result = future.result() + for task in self.helpers.as_completed(tasks): + try: + result, dh = await task + except asyncio.CancelledError: + continue # cancel if we run into timeouts etc. if result is None: @@ -232,14 +235,11 @@ def handle_event(self, event): if fail_count < 2: continue self.debug(f"Cancelling run against {event.data} due to failed request") - for future in futures: - future.cancel() + self.helpers.cancel_tasks(tasks) break - if result: + else: if "Cannot deserialize dialog parameters" in result.text: - for future in futures: - future.cancel() - + self.helpers.cancel_tasks(tasks) self.debug(f"Detected Telerik UI instance ({dh})") description = f"Telerik DialogHandler detected" self.emit_event( @@ -247,16 +247,18 @@ def handle_event(self, event): "FINDING", event, ) - # Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard - break + # Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard + break + + self.helpers.cancel_tasks(tasks) spellcheckhandler = "Telerik.Web.UI.SpellCheckHandler.axd" - result = self.test_detector(event.data, spellcheckhandler) + result, _ = await self.test_detector(event.data, spellcheckhandler) try: # The standard behavior for the spellcheck handler without parameters is a 500 if result.status_code == 500: # Sometimes webapps will just return 500 for everything, so rule out the false positive - validate_result = self.test_detector(event.data, self.helpers.rand_string()) + validate_result, _ = await self.test_detector(event.data, self.helpers.rand_string()) self.debug(validate_result) if validate_result.status_code != 500: self.debug(f"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)") @@ -273,16 +275,16 @@ def handle_event(self, event): except Exception: pass - def test_detector(self, baseurl, detector): + async def test_detector(self, baseurl, detector): result = None if "/" != baseurl[-1]: url = f"{baseurl}/{detector}" else: url = f"{baseurl}{detector}" - result = self.helpers.request(url, timeout=self.timeout) - return result + result = await self.helpers.request(url, timeout=self.timeout) + return result, detector - def filter_event(self, event): + async def filter_event(self, event): if "endpoint" in event.tags: return False else: diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 65a123f3e4..377b690f32 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -182,9 +182,10 @@ async def _emit_event(self, event, *args, **kwargs): await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) # now that the event is properly tagged, we can finally make decisions about it + abort_result = False if callable(abort_if): async with self.scan.acatch(context=abort_if): - abort_result = await self.helpers.execute_sync_or_async(abort_if, event) + abort_result = await self.scan.helpers.execute_sync_or_async(abort_if, event) msg = f"{event.module}: not raising event {event} due to custom criteria in abort_if()" with suppress(ValueError, TypeError): abort_result, reason = abort_result @@ -430,6 +431,17 @@ async def modules_status(self, _log=False): else: self.scan.info(f"{self.scan.name}: No events in queue") + if self.scan.log_level <= logging.DEBUG: + # log module memory usage + module_memory_usage = [] + for module in self.scan.modules.values(): + memory_usage = module.memory_usage + module_memory_usage.append((module.name, memory_usage)) + module_memory_usage.sort(key=lambda x: x[-1], reverse=True) + self.scan.debug(f"MODULE MEMORY USAGE:") + for module_name, usage in module_memory_usage: + self.scan.debug(f" - {module_name}: {self.scan.helpers.bytes_to_human(usage)}") + # Uncomment these lines to enable debugging of event queues # queued_events = self.incoming_event_queue.events diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 80aae3e62b..c43f702a15 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -162,6 +162,9 @@ def __init__( self._cleanedup = False self._loop = asyncio.get_event_loop() + self.manager_worker_loop_tasks = [] + self.init_events_task = None + self.ticker_task = None def _on_keyboard_interrupt(self, loop, event): self.stop() @@ -182,7 +185,7 @@ async def prep(self): start_msg += f" ({', '.join(details)})" self.hugeinfo(start_msg) - self.load_modules() + await self.load_modules() self.info(f"Setting up modules...") await self.setup_modules() @@ -203,7 +206,7 @@ async def start(self): self.warning(f"No scan targets specified") # start status ticker - ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) + self.ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) scan_start_time = datetime.now() try: @@ -219,12 +222,12 @@ async def start(self): self.dispatcher.on_start(self) # start manager worker loops - manager_worker_loop_tasks = [ + self.manager_worker_loop_tasks = [ asyncio.create_task(self.manager._worker_loop()) for _ in range(self.max_workers) ] # distribute seed events - init_events_task = asyncio.create_task(self.manager.init_events()) + self.init_events_task = asyncio.create_task(self.manager.init_events()) self.status = "RUNNING" self.start_modules() @@ -271,10 +274,7 @@ async def start(self): self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}") finally: - init_events_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await init_events_task - + await self.cancel_tasks() await self.report() await self.cleanup() @@ -288,15 +288,6 @@ async def start(self): else: self.status = "FINISHED" - ticker_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await ticker_task - - for t in manager_worker_loop_tasks: - t.cancel() - with contextlib.suppress(asyncio.CancelledError): - await t - scan_run_time = datetime.now() - scan_start_time scan_run_time = self.helpers.human_timedelta(scan_run_time) log_fn(f"Scan {self.name} completed in {scan_run_time} with status {self.status}") @@ -309,7 +300,7 @@ def start_modules(self): module.start() async def setup_modules(self, remove_failed=True): - self.load_modules() + await self.load_modules() self.verbose(f"Setting up modules") hard_failed = [] soft_failed = [] @@ -376,6 +367,24 @@ def drain_queues(self): self.manager.incoming_event_queue.get_nowait() self.debug("Finished draining queues") + async def cancel_tasks(self): + tasks = [] + # module workers + for m in self.modules.values(): + tasks += getattr(m, "_tasks", []) + # init events + if self.init_events_task: + tasks.append(self.init_events_task) + # ticker + if self.ticker_task: + tasks.append(self.ticker_task) + # manager worker loops + tasks += self.manager_worker_loop_tasks + for t in tasks: + t.cancel() + with contextlib.suppress(asyncio.CancelledError): + await t + async def report(self): for mod in self.modules.values(): async with self.acatch(context=mod.report): @@ -540,7 +549,7 @@ def _internal_modules(self): if self.config.get(modname, True): yield modname - def load_modules(self): + async def load_modules(self): if not self._modules_loaded: all_modules = list(set(self._scan_modules + self._output_modules + self._internal_modules)) if not all_modules: @@ -551,7 +560,7 @@ def load_modules(self): self.warning(f"No scan modules to load") # install module dependencies - succeeded, failed = self.helpers.depsinstaller.install( + succeeded, failed = await self.helpers.depsinstaller.install( *self._scan_modules, *self._output_modules, *self._internal_modules ) if failed: diff --git a/bbot/test/helpers.py b/bbot/test/helpers.py index 4654c036ca..fd457795b8 100644 --- a/bbot/test/helpers.py +++ b/bbot/test/helpers.py @@ -1,5 +1,4 @@ import logging -import requests_mock from abc import abstractmethod from omegaconf import OmegaConf @@ -13,11 +12,13 @@ class MockHelper: config_overrides = {} additional_modules = [] - def __init__(self, config, bbot_scanner, *args, **kwargs): + def __init__(self, request, **kwargs): self.name = kwargs.get("module_name", self.__class__.__name__.lower()) - self.config = OmegaConf.merge(config, OmegaConf.create(self.config_overrides)) + self.bbot_config = request.getfixturevalue("bbot_config") + self.bbot_scanner = request.getfixturevalue("bbot_scanner") + self.config = OmegaConf.merge(self.bbot_config, OmegaConf.create(self.config_overrides)) modules = [self.name] + self.additional_modules - self.scan = bbot_scanner( + self.scan = self.bbot_scanner( *self.targets, modules=modules, name=f"{self.name}_test", @@ -25,8 +26,6 @@ def __init__(self, config, bbot_scanner, *args, **kwargs): whitelist=self.whitelist, blacklist=self.blacklist, ) - self.patch_scan(self.scan) - self.setup() def patch_scan(self, scan): return @@ -34,10 +33,13 @@ def patch_scan(self, scan): def setup(self): pass - def run(self): - events = list(self.scan.start()) - events = [e for e in events if e.module == self.module] - assert self.check_events(events) + async def run(self): + await self.scan.prep() + self.setup() + self.patch_scan(self.scan) + self._after_scan_prep() + events = [e async for e in self.scan.start()] + self.check_events(events) @abstractmethod def check_events(self, events): @@ -47,29 +49,29 @@ def check_events(self, events): def module(self): return self.scan.modules[self.name] + def _after_scan_prep(self): + pass + class RequestMockHelper(MockHelper): + def __init__(self, request, **kwargs): + self.httpx_mock = request.getfixturevalue("httpx_mock") + super().__init__(request, **kwargs) + @abstractmethod def mock_args(self): raise NotImplementedError - def register_uri(self, uri, method="GET", **kwargs): - self.m.register_uri(method, uri, **kwargs) - - def run(self): - with requests_mock.Mocker() as m: - self.m = m - self.mock_args() - return super().run() + def _after_scan_prep(self): + self.mock_args() class HttpxMockHelper(MockHelper): targets = ["http://127.0.0.1:8888/"] - def __init__(self, config, bbot_scanner, bbot_httpserver, *args, **kwargs): - self.bbot_httpserver = bbot_httpserver - super().__init__(config, bbot_scanner, *args, **kwargs) - self.mock_args() + def __init__(self, request, **kwargs): + self.bbot_httpserver = request.getfixturevalue("bbot_httpserver") + super().__init__(request, **kwargs) @abstractmethod def mock_args(self): @@ -80,6 +82,9 @@ def set_expect_requests(self, expect_args={}, respond_args={}): expect_args["uri"] = "/" self.bbot_httpserver.expect_request(**expect_args).respond_with_data(**respond_args) + def _after_scan_prep(self): + self.mock_args() + def tempwordlist(content): tmp_path = "/tmp/.bbot_test/" diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index c1bf5bf552..f763318bca 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -8,22 +8,30 @@ class Httpx(HttpxMockHelper): + targets = ["http://127.0.0.1:8888/url", "127.0.0.1:8888"] + def mock_args(self): - request_args = dict(headers={"test": "header"}) - respond_args = dict(response_data=json.dumps({"foo": "bar"})) + request_args = dict(uri="/", headers={"test": "header"}) + respond_args = dict(response_data=json.dumps({"open": "port"})) + self.set_expect_requests(request_args, respond_args) + request_args = dict(uri="/url", headers={"test": "header"}) + respond_args = dict(response_data=json.dumps({"url": "url"})) self.set_expect_requests(request_args, respond_args) - - async def run(self): - events = [e async for e in self.scan.start()] - assert self.check_events(events) def check_events(self, events): + url = False + open_port = False for e in events: if e.type == "HTTP_RESPONSE": j = json.loads(e.data["body"]) - if j.get("foo", "") == "bar": - return True - return False + if e.data["path"] == "/": + if j.get("open", "") == "port": + open_port = True + elif e.data["path"] == "/url": + if j.get("url", "") == "url": + url = True + assert url, "Failed to visit target URL" + assert open_port, "Failed to visit target OPEN_TCP_PORT" class Gowitness(HttpxMockHelper): @@ -63,7 +71,6 @@ def check_events(self, events): assert url, "No URL emitted" assert webscreenshot, "No WEBSCREENSHOT emitted" assert technology, "No TECHNOLOGY emitted" - return True class Excavate(HttpxMockHelper): @@ -158,7 +165,6 @@ def check_events(self, events): and "spider-danger" in e.tags for e in events ) - return True class Excavate_relativelinks(HttpxMockHelper): @@ -206,14 +212,10 @@ def check_events(self, events): if e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": root_page_confusion_2 = True - if ( - root_relative_detection - and page_relative_detection - and not root_page_confusion_1 - and not root_page_confusion_2 - ): - return True - return False + assert root_relative_detection, "Failed to properly excavate root-relative URL" + assert page_relative_detection, "Failed to properly excavate page-relative URL" + assert not root_page_confusion_1, "Incorrectly detected page-relative URL" + assert not root_page_confusion_2, "Incorrectly detected root-relative URL" class Subdomain_Hijack(HttpxMockHelper): @@ -229,15 +231,13 @@ def mock_args(self): self.set_expect_requests(respond_args=respond_args) def check_events(self, events): - for event in events: - if ( - event.type == "FINDING" - and event.data["description"].startswith("Hijackable Subdomain") - and self.rand_subdomain in event.data["description"] - and event.data["host"] == self.rand_subdomain - ): - return True - return False, f"No hijackable subdomains in {events}" + assert any( + event.type == "FINDING" + and event.data["description"].startswith("Hijackable Subdomain") + and self.rand_subdomain in event.data["description"] + and event.data["host"] == self.rand_subdomain + for event in events + ), f"No hijackable subdomains in {events}" class Fingerprintx(HttpxMockHelper): @@ -247,22 +247,20 @@ def mock_args(self): pass def check_events(self, events): - for event in events: - if ( - event.type == "PROTOCOL" - and event.host == self.scan.helpers.make_ip_type("127.0.0.1") - and event.port == 8888 - and event.data["protocol"] == "HTTP" - ): - return True - return False + assert any( + event.type == "PROTOCOL" + and event.host == self.scan.helpers.make_ip_type("127.0.0.1") + and event.port == 8888 + and event.data["protocol"] == "HTTP" + for event in events + ), "HTTP protocol not detected" class Otx(RequestMockHelper): def mock_args(self): for t in self.targets: - self.register_uri( - f"https://otx.alienvault.com/api/v1/indicators/domain/{t}/passive_dns", + self.httpx_mock.add_response( + url=f"https://otx.alienvault.com/api/v1/indicators/domain/{t}/passive_dns", json={ "passive_dns": [ { @@ -282,10 +280,7 @@ def mock_args(self): ) def check_events(self, events): - for e in events: - if e.data == "asdf.blacklanternsecurity.com": - return True - return False + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" class Anubisdb(RequestMockHelper): @@ -294,16 +289,13 @@ def setup(self): def mock_args(self): for t in self.targets: - self.register_uri( - f"https://jldc.me/anubis/subdomains/{t}", + self.httpx_mock.add_response( + url=f"https://jldc.me/anubis/subdomains/{t}", json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], ) def check_events(self, events): - for e in events: - if e.data == "asdf.blacklanternsecurity.com": - return True - return False + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" class SecretsDB(HttpxMockHelper): @@ -315,7 +307,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - return any(e.type == "FINDING" for e in events) + assert any(e.type == "FINDING" for e in events) class Badsecrets(HttpxMockHelper): @@ -421,9 +413,10 @@ def check_events(self, events): ): CookieBasedDetection_2 = True - if SecretFound and IdentifyOnly and CookieBasedDetection and CookieBasedDetection_2: - return True - return False + assert SecretFound, "No secret found" + assert IdentifyOnly, "No crypto product identified" + assert CookieBasedDetection, "No JWT cookie detected" + assert CookieBasedDetection_2, "No Express.js cookie detected" class Telerik(HttpxMockHelper): @@ -491,18 +484,14 @@ def check_events(self, events): telerik_spellcheck_detection = True continue - if ( - telerik_axd_detection - and telerik_axd_vulnerable - and telerik_spellcheck_detection - and telerik_dialoghandler_detection - ): - return True - return False + assert telerik_axd_detection, "Telerik AXD detection failed" + assert telerik_axd_vulnerable, "Telerik vulnerable AXD detection failed" + assert telerik_spellcheck_detection, "Telerik spellcheck detection failed" + assert telerik_dialoghandler_detection, "Telerik dialoghandler detection failed" -class Paramminer_getparams(HttpxMockHelper): - getparam_body = """ +class Paramminer_headers(HttpxMockHelper): + headers_body = """ the title @@ -511,7 +500,7 @@ class Paramminer_getparams(HttpxMockHelper): """ - getparam_body_match = """ + headers_body_match = """ the title @@ -521,7 +510,7 @@ class Paramminer_getparams(HttpxMockHelper): """ additional_modules = ["httpx"] - config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist(["canary", "id"])}}} + config_overrides = {"modules": {"paramminer_headers": {"wordlist": tempwordlist(["junkword1", "tracestate"])}}} def setup(self): from bbot.core.helpers import helper @@ -530,22 +519,26 @@ def setup(self): helper.HttpCompare.gen_cache_buster = lambda *args, **kwargs: {"AAAAAA": "1"} def mock_args(self): - expect_args = {"query_string": b"id=AAAAAAAAAAAAAA&AAAAAA=1"} - respond_args = {"response_data": self.getparam_body_match} + expect_args = dict(headers={"tracestate": "AAAAAAAAAAAAAA"}) + respond_args = {"response_data": self.headers_body_match} self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - respond_args = {"response_data": self.getparam_body} + respond_args = {"response_data": self.headers_body} self.set_expect_requests(respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [id] Reasons: [body]": - return True - return False + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [tracestate] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [junkword1] Reasons: [body]" + for e in events + ) -class Paramminer_headers(HttpxMockHelper): - headers_body = """ +class Paramminer_getparams(HttpxMockHelper): + getparam_body = """ the title @@ -554,7 +547,7 @@ class Paramminer_headers(HttpxMockHelper): """ - headers_body_match = """ + getparam_body_match = """ the title @@ -564,7 +557,7 @@ class Paramminer_headers(HttpxMockHelper): """ additional_modules = ["httpx"] - config_overrides = {"modules": {"paramminer_headers": {"wordlist": tempwordlist(["junkword1", "tracestate"])}}} + config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist(["canary", "id"])}}} def setup(self): from bbot.core.helpers import helper @@ -573,18 +566,22 @@ def setup(self): helper.HttpCompare.gen_cache_buster = lambda *args, **kwargs: {"AAAAAA": "1"} def mock_args(self): - expect_args = dict(headers={"tracestate": "AAAAAAAAAAAAAA"}) - respond_args = {"response_data": self.headers_body_match} + expect_args = {"query_string": b"id=AAAAAAAAAAAAAA&AAAAAA=1"} + respond_args = {"response_data": self.getparam_body_match} self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - respond_args = {"response_data": self.headers_body} + respond_args = {"response_data": self.getparam_body} self.set_expect_requests(respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [tracestate] Reasons: [body]": - return True - return False + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [id] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [canary] Reasons: [body]" + for e in events + ) class Paramminer_cookies(HttpxMockHelper): @@ -624,16 +621,20 @@ def mock_args(self): self.set_expect_requests(respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [admincookie] Reasons: [body]": - return True - return False + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [admincookie] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [junkcookie] Reasons: [body]" + for e in events + ) class LeakIX(RequestMockHelper): def mock_args(self): - self.register_uri( - "https://leakix.net/api/subdomains/blacklanternsecurity.com", + self.httpx_mock.add_response( + url="https://leakix.net/api/subdomains/blacklanternsecurity.com", json=[ { "subdomain": "www.blacklanternsecurity.com", @@ -652,32 +653,33 @@ def check_events(self, events): www = False asdf = False for e in events: - if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED"): + if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "leakix": if e.data == "www.blacklanternsecurity.com": www = True elif e.data == "asdf.blacklanternsecurity.com": asdf = True - return www and asdf + assert www + assert asdf -class Massdns(MockHelper): +class Massdns(RequestMockHelper): subdomain_wordlist = tempwordlist(["www", "asdf"]) config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist)}}} - def __init__(self, *args, **kwargs): - with requests_mock.Mocker() as m: - m.register_uri( - "GET", - "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt", - text="8.8.8.8\n8.8.4.4\n1.1.1.1", - ) - super().__init__(*args, **kwargs) + def __init__(self, request): + super().__init__(request) + self.httpx_mock.add_response( + url="https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt", + text="8.8.8.8\n8.8.4.4\n1.1.1.1", + ) + + def mock_args(self): + pass def check_events(self, events): - for e in events: - if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and e.data == "www.blacklanternsecurity.com": - return True - return False + assert any( + e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and e.data == "www.blacklanternsecurity.com" for e in events + ) class Robots(HttpxMockHelper): @@ -700,7 +702,8 @@ def check_events(self, events): for e in events: if e.type == "URL_UNVERIFIED": - assert "spider-danger" in e.tags, f"{e} doesn't have spider-danger tag" + if str(e.module) != "TARGET": + assert "spider-danger" in e.tags, f"{e} doesn't have spider-danger tag" if e.data == "http://127.0.0.1:8888/allow/": allow_bool = True @@ -713,9 +716,10 @@ def check_events(self, events): if re.match(r"http://127\.0\.0\.1:8888/\w+/wildcard\.txt", e.data): wildcard_bool = True - if allow_bool and disallow_bool and sitemap_bool and wildcard_bool: - return True - return False + assert allow_bool + assert disallow_bool + assert sitemap_bool + assert wildcard_bool class Masscan(MockHelper): @@ -734,13 +738,12 @@ class Masscan(MockHelper): ports = range = 9.8.7.6""" - def __init__(self, config, bbot_scanner, *args, **kwargs): - super().__init__(config, bbot_scanner, *args, **kwargs) + def _after_scan_prep(self): self.scan.modules["masscan"].masscan_config = self.masscan_config - def setup_scan_2(): - config2 = OmegaConf.merge(config, OmegaConf.create(self.config_overrides_2)) - self.scan2 = bbot_scanner( + async def setup_scan_2(): + config2 = OmegaConf.merge(self.config, OmegaConf.create(self.config_overrides_2)) + self.scan2 = self.bbot_scanner( *self.targets, modules=[self.name] + self.additional_modules, name=f"{self.name}_test", @@ -749,35 +752,37 @@ def setup_scan_2(): blacklist=self.blacklist, ) self.patch_scan(self.scan2) - self.scan2.prep() + await self.scan2.prep() self.scan2.modules["masscan"].masscan_config = self.masscan_config self.setup_scan_2 = setup_scan_2 self.masscan_run = False - def run_masscan(self, command, *args, **kwargs): - if "masscan" in command[0]: - yield from self.masscan_output.splitlines() + async def run_masscan(self, command, *args, **kwargs): + log.critical(f"patched: {command}") + if "masscan" in command[:2]: + for l in self.masscan_output.splitlines(): + yield l self.masscan_run = True else: - yield from self.scan.helpers.run_live(command, *args, **kwargs) + async for l in self.scan.helpers.run_live(command, *args, **kwargs): + yield l def patch_scan(self, scan): scan.helpers.run_live = self.run_masscan - def run(self): - super().run() - self.setup_scan_2() + async def run(self): + await super().run() assert self.masscan_run == True, "masscan didn't run when it was supposed to" + await self.setup_scan_2() self.masscan_run = False - events = list(self.scan2.start()) + events = [e async for e in self.scan2.start()] self.check_events(events) assert self.masscan_run == False, "masscan ran when it wasn't supposed to" def check_events(self, events): assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" - return True class Buckets(HttpxMockHelper, RequestMockHelper): diff --git a/bbot/test/test_step_1/test_modules_full.py b/bbot/test/test_step_1/test_modules_full.py index 8d1f58204b..3dea25d310 100644 --- a/bbot/test/test_step_1/test_modules_full.py +++ b/bbot/test/test_step_1/test_modules_full.py @@ -6,192 +6,229 @@ log = logging.getLogger(f"bbot.test") -def test_gowitness(bbot_config, bbot_scanner, bbot_httpserver): - x = Gowitness(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_httpx(request): + x = Httpx(request) + await x.run() @pytest.mark.asyncio -async def test_httpx(bbot_config, bbot_scanner, bbot_httpserver): - x = Httpx(bbot_config, bbot_scanner, bbot_httpserver) +async def test_gowitness(request): + x = Gowitness(request) await x.run() -def test_excavate(bbot_config, bbot_scanner, bbot_httpserver): - x = Excavate(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_excavate(request): + x = Excavate(request) + await x.run() -def test_excavate_relativelinks(bbot_config, bbot_scanner, bbot_httpserver): - x = Excavate_relativelinks(bbot_config, bbot_scanner, bbot_httpserver, module_name="excavate") - x.run() +@pytest.mark.asyncio +async def test_excavate_relativelinks(request): + x = Excavate_relativelinks(request, module_name="excavate") + await x.run() -def test_subdomain_hijack(bbot_config, bbot_scanner, bbot_httpserver): - x = Subdomain_Hijack(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_subdomain_hijack(request): + x = Subdomain_Hijack(request) + await x.run() -def test_fingerprintx(bbot_config, bbot_scanner, bbot_httpserver): - x = Fingerprintx(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_fingerprintx(request): + x = Fingerprintx(request) + await x.run() -def test_otx(bbot_config, bbot_scanner, bbot_httpserver): - x = Otx(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_otx(request): + x = Otx(request) + await x.run() -def test_anubisdb(bbot_config, bbot_scanner, bbot_httpserver): - x = Anubisdb(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_anubisdb(request): + x = Anubisdb(request) + await x.run() -def test_paramminer_getparams(bbot_config, bbot_scanner, bbot_httpserver): - x = Paramminer_getparams(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_secretsdb(request): + x = SecretsDB(request) + await x.run() -def test_paramminer_headers(bbot_config, bbot_scanner, bbot_httpserver): - x = Paramminer_headers(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_badsecrets(request): + x = Badsecrets(request) + await x.run() -def test_paramminer_cookies(bbot_config, bbot_scanner, bbot_httpserver): - x = Paramminer_cookies(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_telerik(request): + x = Telerik(request) + await x.run() -def test_telerik(bbot_config, bbot_scanner, bbot_httpserver): - x = Telerik(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_paramminer_headers(request): + x = Paramminer_headers(request) + await x.run() -def test_leakix(bbot_config, bbot_scanner, bbot_httpserver): - x = LeakIX(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_paramminer_getparams(request): + x = Paramminer_getparams(request) + await x.run() -def test_massdns(bbot_config, bbot_scanner, bbot_httpserver): - x = Massdns(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_paramminer_cookies(request): + x = Paramminer_cookies(request) + await x.run() -def test_masscan(bbot_config, bbot_scanner, bbot_httpserver): - x = Masscan(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_leakix(request): + x = LeakIX(request) + await x.run() -def test_secretsdb(bbot_config, bbot_scanner, bbot_httpserver): - x = SecretsDB(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_massdns(request): + x = Massdns(request) + await x.run() -def test_badsecrets(bbot_config, bbot_scanner, bbot_httpserver): - x = Badsecrets(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_masscan(request): + x = Masscan(request) + await x.run() -def test_robots(bbot_config, bbot_scanner, bbot_httpserver): - x = Robots(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_robots(request): + x = Robots(request) + await x.run() -def test_buckets(bbot_config, bbot_scanner, bbot_httpserver): - x = Buckets(bbot_config, bbot_scanner, bbot_httpserver, module_name="excavate") - x.run() +@pytest.mark.asyncio +async def test_buckets(request): + x = Buckets(request, module_name="excavate") + await x.run() -def test_asn(bbot_config, bbot_scanner, bbot_httpserver): - x = ASN(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_asn(request): + x = ASN(request) + await x.run() -def test_wafw00f(bbot_config, bbot_scanner, bbot_httpserver): - x = Wafw00f(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_wafw00f(request): + x = Wafw00f(request) + await x.run() -def test_ffuf(bbot_config, bbot_scanner, bbot_httpserver): - x = Ffuf(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_ffuf(request): + x = Ffuf(request) + await x.run() -def test_ffuf_extensions(bbot_config, bbot_scanner, bbot_httpserver): - x = Ffuf_extensions(bbot_config, bbot_scanner, bbot_httpserver, module_name="ffuf") - x.run() +@pytest.mark.asyncio +async def test_ffuf_extensions(request): + x = Ffuf_extensions(request, module_name="ffuf") + await x.run() -def test_bypass403(bbot_config, bbot_scanner, bbot_httpserver): - x = Bypass403(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_bypass403(request): + x = Bypass403(request) + await x.run() -def test_bypass403_waf(bbot_config, bbot_scanner, bbot_httpserver): - x = Bypass403_waf(bbot_config, bbot_scanner, bbot_httpserver, module_name="bypass403") - x.run() +@pytest.mark.asyncio +async def test_bypass403_waf(request): + x = Bypass403_waf(request, module_name="bypass403") + await x.run() -def test_bypass403_aspnetcookieless(bbot_config, bbot_scanner, bbot_httpserver): - x = Bypass403_aspnetcookieless(bbot_config, bbot_scanner, bbot_httpserver, module_name="bypass403") - x.run() +@pytest.mark.asyncio +async def test_bypass403_aspnetcookieless(request): + x = Bypass403_aspnetcookieless(request, module_name="bypass403") + await x.run() -def test_ffuf_shortnames(bbot_config, bbot_scanner, bbot_httpserver): - x = Ffuf_shortnames(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_ffuf_shortnames(request): + x = Ffuf_shortnames(request) + await x.run() -def test_iis_shortnames(bbot_config, bbot_scanner, bbot_httpserver): - x = Iis_shortnames(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_iis_shortnames(request): + x = Iis_shortnames(request) + await x.run() -def test_nuclei_technology(bbot_config, bbot_scanner, bbot_httpserver, caplog): - x = Nuclei_technology(bbot_config, bbot_scanner, bbot_httpserver, caplog, module_name="nuclei") - x.run() +@pytest.mark.asyncio +async def test_nuclei_technology(request, caplog): + x = Nuclei_technology(request, caplog, module_name="nuclei") + await x.run() -def test_nuclei_manual(bbot_config, bbot_scanner, bbot_httpserver): - x = Nuclei_manual(bbot_config, bbot_scanner, bbot_httpserver, module_name="nuclei") - x.run() +@pytest.mark.asyncio +async def test_nuclei_manual(request): + x = Nuclei_manual(request, module_name="nuclei") + await x.run() -def test_nuclei_severe(bbot_config, bbot_scanner, bbot_httpserver): - x = Nuclei_severe(bbot_config, bbot_scanner, bbot_httpserver, module_name="nuclei") - x.run() +@pytest.mark.asyncio +async def test_nuclei_severe(request): + x = Nuclei_severe(request, module_name="nuclei") + await x.run() -def test_nuclei_budget(bbot_config, bbot_scanner, bbot_httpserver): - x = Nuclei_budget(bbot_config, bbot_scanner, bbot_httpserver, module_name="nuclei") - x.run() +@pytest.mark.asyncio +async def test_nuclei_budget(request): + x = Nuclei_budget(request, module_name="nuclei") + await x.run() -def test_url_manipulation(bbot_config, bbot_scanner, bbot_httpserver): - x = Url_manipulation(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_url_manipulation(request): + x = Url_manipulation(request) + await x.run() -def test_naabu(bbot_config, bbot_scanner, bbot_httpserver): - x = Naabu(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_naabu(request): + x = Naabu(request) + await x.run() -def test_hunt(bbot_config, bbot_scanner, bbot_httpserver): - x = Hunt(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_hunt(request): + x = Hunt(request) + await x.run() -def test_vhost(bbot_config, bbot_scanner, bbot_httpserver): - x = Vhost(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_vhost(request): + x = Vhost(request) + await x.run() -def test_speculate_subdirectories(bbot_config, bbot_scanner, bbot_httpserver): - x = Speculate_subdirectories(bbot_config, bbot_scanner, bbot_httpserver, module_name="speculate") - x.run() +@pytest.mark.asyncio +async def test_speculate_subdirectories(request): + x = Speculate_subdirectories(request, module_name="speculate") + await x.run() -def test_social(bbot_config, bbot_scanner, bbot_httpserver): - x = Social(bbot_config, bbot_scanner, bbot_httpserver) - x.run() +@pytest.mark.asyncio +async def test_social(request): + x = Social(request) + await x.run() diff --git a/bbot/test/test_step_2/test_command.py b/bbot/test/test_step_2/test_command.py index 5f56d248b0..b45039c319 100644 --- a/bbot/test/test_step_2/test_command.py +++ b/bbot/test/test_step_2/test_command.py @@ -1,4 +1,5 @@ from ..bbot_fixtures import * +from subprocess import CalledProcessError @pytest.mark.asyncio @@ -7,10 +8,15 @@ async def test_command(bbot_scanner, bbot_config): # run assert "plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"])).stdout + assert b"plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"], text=False)).stdout result = (await scan1.helpers.run(["cat"], input="some\nrandom\nstdin")).stdout assert result.splitlines() == ["some", "random", "stdin"] + result = (await scan1.helpers.run(["cat"], input=b"some\nrandom\nstdin", text=False)).stdout + assert result.splitlines() == [b"some", b"random", b"stdin"] result = (await scan1.helpers.run(["cat"], input=["some", "random", "stdin"])).stdout assert result.splitlines() == ["some", "random", "stdin"] + result = (await scan1.helpers.run(["cat"], input=[b"some", b"random", b"stdin"], text=False)).stdout + assert result.splitlines() == [b"some", b"random", b"stdin"] # run_live lines = [] @@ -18,6 +24,10 @@ async def test_command(bbot_scanner, bbot_config): lines.append(line) assert lines == ["plumbus"] lines = [] + async for line in scan1.helpers.run_live(["echo", "plumbus"], text=False): + lines.append(line) + assert lines == [b"plumbus"] + lines = [] async for line in scan1.helpers.run_live(["cat"], input="some\nrandom\nstdin"): lines.append(line) assert lines == ["some", "random", "stdin"] @@ -26,6 +36,20 @@ async def test_command(bbot_scanner, bbot_config): lines.append(line) assert lines == ["some", "random", "stdin"] + # test check=True + with pytest.raises(CalledProcessError) as excinfo: + lines = [l async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True)] + assert "No such file or directory" in excinfo.value.stderr + with pytest.raises(CalledProcessError) as excinfo: + lines = [l async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True, text=False)] + assert b"No such file or directory" in excinfo.value.stderr + with pytest.raises(CalledProcessError) as excinfo: + await scan1.helpers.run(["ls", "/aslkdjflasdkfsd"], check=True) + assert "No such file or directory" in excinfo.value.stderr + with pytest.raises(CalledProcessError) as excinfo: + await scan1.helpers.run(["ls", "/aslkdjflasdkfsd"], check=True, text=False) + assert b"No such file or directory" in excinfo.value.stderr + # test piping lines = [] async for line in scan1.helpers.run_live( @@ -33,6 +57,12 @@ async def test_command(bbot_scanner, bbot_config): ): lines.append(line) assert lines == ["some", "random", "stdin"] + lines = [] + async for line in scan1.helpers.run_live( + ["cat"], input=scan1.helpers.run_live(["echo", "-en", r"some\nrandom\nstdin"], text=False), text=False + ): + lines.append(line) + assert lines == [b"some", b"random", b"stdin"] # test missing executable result = await scan1.helpers.run(["sgkjlskdfsdf"]) @@ -50,25 +80,25 @@ async def test_command(bbot_scanner, bbot_config): path_parts = os.environ.get("PATH", "").split(":") assert "/tmp/.bbot_test/tools" in path_parts run_lines = (await scan1.helpers.run(["env"])).stdout.splitlines() - assert f"BBOT_PLUMBUS=asdf" in run_lines + assert "BBOT_PLUMBUS=asdf" in run_lines for line in run_lines: if line.startswith("PATH="): path_parts = line.split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_lines_sudo = (await scan1.helpers.run(["env"], sudo=True)).stdout.splitlines() - assert f"BBOT_PLUMBUS=asdf" in run_lines_sudo + assert "BBOT_PLUMBUS=asdf" in run_lines_sudo for line in run_lines_sudo: if line.startswith("PATH="): path_parts = line.split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_live_lines = [l async for l in scan1.helpers.run_live(["env"])] - assert f"BBOT_PLUMBUS=asdf" in run_live_lines + assert "BBOT_PLUMBUS=asdf" in run_live_lines for line in run_live_lines: if line.startswith("PATH="): path_parts = line.strip().split("=", 1)[-1].split(":") assert "/tmp/.bbot_test/tools" in path_parts run_live_lines_sudo = [l async for l in scan1.helpers.run_live(["env"], sudo=True)] - assert f"BBOT_PLUMBUS=asdf" in run_live_lines_sudo + assert "BBOT_PLUMBUS=asdf" in run_live_lines_sudo for line in run_live_lines_sudo: if line.startswith("PATH="): path_parts = line.strip().split("=", 1)[-1].split(":") diff --git a/poetry.lock b/poetry.lock index 3141b7e3fc..146a03dd54 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1230,6 +1230,25 @@ files = [ [package.dependencies] Werkzeug = ">=2.0.0" +[[package]] +name = "pytest-httpx" +version = "0.22.0" +description = "Send responses to httpx." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest_httpx-0.22.0-py3-none-any.whl", hash = "sha256:cefb7dcf66a4cb0601b0de05e576cca423b6081f3245e7912a4d84c58fa3eae8"}, + {file = "pytest_httpx-0.22.0.tar.gz", hash = "sha256:3a82797f3a9a14d51e8c6b7fa97524b68b847ee801109c062e696b4744f4431c"}, +] + +[package.dependencies] +httpx = ">=0.24.0,<0.25.0" +pytest = ">=6.0,<8.0" + +[package.extras] +testing = ["pytest-asyncio (>=0.20.0,<0.21.0)", "pytest-cov (>=4.0.0,<5.0.0)"] + [[package]] name = "pytest-rerunfailures" version = "11.1.2" @@ -1386,26 +1405,6 @@ files = [ requests = ">=1.0.0" six = "*" -[[package]] -name = "requests-mock" -version = "1.10.0" -description = "Mock out responses from the requests package" -category = "dev" -optional = false -python-versions = "*" -files = [ - {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, - {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, -] - -[package.dependencies] -requests = ">=2.3,<3" -six = "*" - -[package.extras] -fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] - [[package]] name = "resolvelib" version = "0.8.1" @@ -1705,4 +1704,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5366646b07e8aa2c1189f64e976bab2ec624855eb5943bde2710376c8e115305" +content-hash = "e8bc6b97f09142c3d7099d7378d826270a470ee7b454bd39f8c52779d57153cf" diff --git a/pyproject.toml b/pyproject.toml index d8b6f50fc0..fb77583d86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,11 +35,11 @@ pytest = "^7.2.2" flake8 = "^6.0.0" black = "^23.1.0" pytest-cov = "^4.0.0" -requests-mock = "^1.10.0" poetry-dynamic-versioning = "^0.21.4" pytest-httpserver = "^1.0.6" pytest-rerunfailures = "^11.1.2" pytest-asyncio = "^0.21.0" +pytest-httpx = "^0.22.0" [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] From fc5ab4aba5134dd01b8fb6795652c3b1152eabbb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 12 May 2023 16:57:33 -0400 Subject: [PATCH 013/387] asyncified more tests --- bbot/core/helpers/cloud/__init__.py | 8 +- bbot/core/helpers/cloud/base.py | 34 ++++- bbot/core/helpers/dns.py | 24 ++-- bbot/core/helpers/misc.py | 10 +- bbot/core/helpers/web.py | 20 ++- bbot/modules/bucket_aws.py | 50 ++++--- bbot/modules/bucket_azure.py | 6 +- bbot/modules/bucket_firebase.py | 8 +- bbot/modules/bucket_gcp.py | 10 +- bbot/modules/internal/excavate.py | 9 +- bbot/modules/internal/speculate.py | 14 +- bbot/modules/masscan.py | 1 - bbot/modules/report/asn.py | 36 +++--- bbot/modules/wafw00f.py | 13 +- bbot/scanner/scanner.py | 4 +- bbot/test/helpers.py | 25 ++-- bbot/test/modules_test_classes.py | 194 +++++++++++++--------------- 17 files changed, 253 insertions(+), 213 deletions(-) diff --git a/bbot/core/helpers/cloud/__init__.py b/bbot/core/helpers/cloud/__init__.py index 6c682d184a..e8fbf69e3b 100644 --- a/bbot/core/helpers/cloud/__init__.py +++ b/bbot/core/helpers/cloud/__init__.py @@ -25,9 +25,13 @@ def __init__(self, parent_helper): self.providers[provider_name] = provider setattr(self, provider_name, provider) - def excavate(self, event, http_body): + def excavate(self, *args, **kwargs): for provider in self.providers.values(): - provider.excavate(event, http_body) + provider.excavate(*args, **kwargs) + + def speculate(self, *args, **kwargs): + for provider in self.providers.values(): + provider.speculate(*args, **kwargs) def __iter__(self): yield from self.providers.values() diff --git a/bbot/core/helpers/cloud/base.py b/bbot/core/helpers/cloud/base.py index 0e7d26ee94..bd7b8b236a 100644 --- a/bbot/core/helpers/cloud/base.py +++ b/bbot/core/helpers/cloud/base.py @@ -41,6 +41,25 @@ def excavate(self, event, http_body): else: self.emit_event(**kwargs) + def speculate(self, event): + base_kwargs = dict(source=event, tags=self.base_tags) + + if event.type.startswith("DNS_NAME"): + # check for DNS_NAMEs that are buckets + for event_type, sigs in self.signatures.items(): + found = set() + for sig in sigs: + match = sig.match(event.data) + if match: + kwargs = dict(base_kwargs) + kwargs["event_type"] = event_type + if not event.data in found: + found.add(event.data) + if event_type == "STORAGE_BUCKET": + self.emit_bucket(match.groups(), **kwargs) + else: + self.emit_event(**kwargs) + def emit_bucket(self, match, **kwargs): bucket_name, bucket_domain = match kwargs["data"] = {"name": bucket_name, "url": f"https://{bucket_name}.{bucket_domain}"} @@ -61,11 +80,16 @@ def tag_event(self, event): # its host directly matches this cloud provider's domains if isinstance(event.host, str) and self.domain_match(event.host): event.tags.update(self.base_tags) - return - # or it has a CNAME that matches this cloud provider's domains - for rh in event.resolved_hosts: - if not self.parent_helper.is_ip(rh) and self.domain_match(rh): - event.tags.update(self.base_tags) + # tag as buckets, etc. + for event_type, sigs in self.signatures.items(): + for sig in sigs: + if sig.match(event.host): + event.add_tag(f"cloud-{event_type}") + else: + # or it has a CNAME that matches this cloud provider's domains + for rh in event.resolved_hosts: + if not self.parent_helper.is_ip(rh) and self.domain_match(rh): + event.tags.update(self.base_tags) def domain_match(self, s): for r in self.domain_regexes: diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 23fd8601f8..4f4b651199 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -219,7 +219,7 @@ async def handle_wildcard_event(self, event, children): event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if not is_ip(event.host) and children and wildcard_rdtypes: + if not is_ip(event.host) and wildcard_rdtypes and children: # these are the rdtypes that successfully resolve resolved_rdtypes = set([c.upper() for c in children]) # these are the rdtypes that have wildcards @@ -228,16 +228,18 @@ async def handle_wildcard_event(self, event, children): event_is_wildcard = False if resolved_rdtypes: event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes) - if event_is_wildcard and event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = self.parent_helper.parent_domain(event_host) - for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): - if _is_wildcard: - wildcard_parent = _parent_domain - break - wildcard_data = f"_wildcard.{wildcard_parent}" - if wildcard_data != event.data: - log.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') - event.data = wildcard_data + # if event_is_wildcard and event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): + if event_is_wildcard: + if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): + wildcard_parent = self.parent_helper.parent_domain(event_host) + for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): + if _is_wildcard: + wildcard_parent = _parent_domain + break + wildcard_data = f"_wildcard.{wildcard_parent}" + if wildcard_data != event.data: + log.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') + event.data = wildcard_data async def resolve_event(self, event, minimal=False): """ diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 243901231f..c2b39214a7 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -19,7 +19,6 @@ import traceback import subprocess as sp from pathlib import Path -from asyncio import sleep # noqa from itertools import islice from datetime import datetime from tabulate import tabulate @@ -28,6 +27,7 @@ import cloudcheck as _cloudcheck import tldextract as _tldextract from hashlib import sha1 as hashlib_sha1 +from asyncio import as_completed, create_task, sleep # noqa from urllib.parse import urlparse, quote, unquote, urlunparse # noqa F401 from .url import * # noqa F401 @@ -1056,14 +1056,6 @@ def get_traceback_details(e): return filename, lineno, funcname -def create_task(*args, **kwargs): - return asyncio.create_task(*args, **kwargs) - - -def as_completed(*args, **kwargs): - yield from asyncio.as_completed(*args, **kwargs) - - async def cancel_tasks(tasks): for task in tasks: task.cancel() diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index ba642d3554..606cca4d3f 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -44,19 +44,32 @@ class WebHelper: For making HTTP requests """ - client_options = ("auth", "params", "headers", "cookies", "timeout", "follow_redirects", "max_redirects") + client_options = ( + "auth", + "params", + "headers", + "retries", + "cookies", + "verify", + "timeout", + "follow_redirects", + "max_redirects", + ) def __init__(self, parent_helper): self.parent_helper = parent_helper + self.ssl_verify = self.parent_helper.config.get("ssl_verify", False) def AsyncClient(self, *args, **kwargs): kwargs["_bbot_scan"] = self.parent_helper.scan retries = kwargs.pop("retries", self.parent_helper.config.get("http_retries", 1)) - kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries) + kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify) return BBOTAsyncClient(*args, **kwargs) async def request(self, *args, **kwargs): raise_error = kwargs.pop("raise_error", False) + # TODO: use this + cache_for = kwargs.pop("cache_for", None) # noqa # in case of URL only, assume GET request if len(args) == 1: @@ -183,8 +196,7 @@ async def curl(self, *args, **kwargs): curl_command.append("--path-as-is") # respect global ssl verify settings - ssl_verify = self.parent_helper.config.get("ssl_verify") - if ssl_verify == False: + if self.ssl_verify == False: curl_command.append("-k") headers = kwargs.get("headers", {}) diff --git a/bbot/modules/bucket_aws.py b/bbot/modules/bucket_aws.py index 4d72ef611c..012bb6df6f 100644 --- a/bbot/modules/bucket_aws.py +++ b/bbot/modules/bucket_aws.py @@ -19,13 +19,13 @@ class bucket_aws(BaseModule): regions = [None] supports_open_check = True - def setup(self): + async def setup(self): self.buckets_tried = set() self.cloud_helper = getattr(self.helpers.cloud, self.cloud_helper_name) self.permutations = self.config.get("permutations", False) return True - def filter_event(self, event): + async def filter_event(self, event): if event.type == "DNS_NAME" and event.scope_distance > 0: return False, "only accepts in-scope DNS_NAMEs" if event.type == "STORAGE_BUCKET": @@ -33,13 +33,13 @@ def filter_event(self, event): return False, "bucket belongs to a different cloud provider" return True - def handle_event(self, event): + async def handle_event(self, event): if event.type == "DNS_NAME": - self.handle_dns_name(event) + await self.handle_dns_name(event) elif event.type == "STORAGE_BUCKET": - self.handle_storage_bucket(event) + await self.handle_storage_bucket(event) - def handle_dns_name(self, event): + async def handle_dns_name(self, event): buckets = set() base = event.data stem = self.helpers.domain_stem(base) @@ -47,24 +47,24 @@ def handle_dns_name(self, event): split = b.split(".") for d in self.delimiters: buckets.add(d.join(split)) - for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations): + async for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations): self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) - def handle_storage_bucket(self, event): + async def handle_storage_bucket(self, event): url = event.data["url"] bucket_name = event.data["name"] if self.supports_open_check: - description, tags = self._check_bucket_open(bucket_name, url) + description, tags = await self._check_bucket_open(bucket_name, url) if description: event_data = {"host": event.host, "url": url, "description": description} self.emit_event(event_data, "FINDING", source=event, tags=tags) - for bucket_name, url, tags in self.brute_buckets( + async for bucket_name, url, tags in self.brute_buckets( [bucket_name], permutations=self.permutations, omit_base=True ): self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) - def brute_buckets(self, buckets, permutations=False, omit_base=False): + async def brute_buckets(self, buckets, permutations=False, omit_base=False): buckets = set(buckets) new_buckets = set(buckets) if permutations: @@ -75,36 +75,34 @@ def brute_buckets(self, buckets, permutations=False, omit_base=False): if omit_base: new_buckets = new_buckets - buckets new_buckets = [b for b in new_buckets if self.valid_bucket_name(b)] - futures = {} + tasks = [] for base_domain in self.base_domains: for region in self.regions: for bucket_name in new_buckets: url = self.build_url(bucket_name, base_domain, region) - future = self.submit_task(self._check_bucket_exists, bucket_name, url) - futures[future] = (bucket_name, url) - for future in self.helpers.as_completed(futures): - bucket_name, url = futures[future] - existent_bucket, tags = future.result() + tasks.append(self.helpers.create_task(self._check_bucket_exists(bucket_name, url))) + for task in self.helpers.as_completed(tasks): + existent_bucket, tags, bucket_name, url = await task if existent_bucket: yield bucket_name, url, tags - def _check_bucket_exists(self, bucket_name, url): + async def _check_bucket_exists(self, bucket_name, url): self.debug(f'Checking if bucket exists: "{bucket_name}"') - return self.check_bucket_exists(bucket_name, url) + return await self.check_bucket_exists(bucket_name, url) - def check_bucket_exists(self, bucket_name, url): - response = self.helpers.request(url) + async def check_bucket_exists(self, bucket_name, url): + response = await self.helpers.request(url) tags = self.gen_tags_exists(response) status_code = getattr(response, "status_code", 404) existent_bucket = status_code != 404 - return (existent_bucket, tags) + return (existent_bucket, tags, bucket_name, url) - def _check_bucket_open(self, bucket_name, url): + async def _check_bucket_open(self, bucket_name, url): self.debug(f'Checking if bucket is misconfigured: "{bucket_name}"') - return self.check_bucket_open(bucket_name, url) + return await self.check_bucket_open(bucket_name, url) - def check_bucket_open(self, bucket_name, url): - response = self.helpers.request(url) + async def check_bucket_open(self, bucket_name, url): + response = await self.helpers.request(url) tags = self.gen_tags_exists(response) status_code = getattr(response, "status_code", 404) content = getattr(response, "text", "") diff --git a/bbot/modules/bucket_azure.py b/bbot/modules/bucket_azure.py index 64d20d1544..97d81fa231 100644 --- a/bbot/modules/bucket_azure.py +++ b/bbot/modules/bucket_azure.py @@ -18,9 +18,9 @@ class bucket_azure(bucket_aws): # Dirbusting is required to know whether a bucket is public supports_open_check = False - def check_bucket_exists(self, bucket_name, url): + async def check_bucket_exists(self, bucket_name, url): url = url.strip("/") + f"/{bucket_name}?restype=container" - response = self.helpers.request(url, retries=0) + response = await self.helpers.request(url, retries=0) status_code = getattr(response, "status_code", 0) existent_bucket = status_code != 0 - return (existent_bucket, set()) + return existent_bucket, set(), bucket_name, url diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index daab455dba..413457893a 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -16,13 +16,13 @@ class bucket_firebase(bucket_aws): delimiters = ("", "-") base_domains = ["firebaseio.com"] - def check_bucket_exists(self, bucket_name, url): + async def check_bucket_exists(self, bucket_name, url): url = url.strip("/") + "/.json" - return super().check_bucket_exists(bucket_name, url) + return await super().check_bucket_exists(bucket_name, url) - def check_bucket_open(self, bucket_name, url): + async def check_bucket_open(self, bucket_name, url): url = url.strip("/") + "/.json" - response = self.helpers.request(url) + response = await self.helpers.request(url) tags = self.gen_tags_exists(response) status_code = getattr(response, "status_code", 404) msg = "" diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py index 5a2c2d7bc0..06cc588946 100644 --- a/bbot/modules/bucket_gcp.py +++ b/bbot/modules/bucket_gcp.py @@ -29,12 +29,12 @@ class bucket_gcp(bucket_aws): def build_url(self, bucket_name, base_domain, region): return f"https://www.googleapis.com/storage/v1/b/{bucket_name}" - def check_bucket_open(self, bucket_name, url): + async def check_bucket_open(self, bucket_name, url): bad_permissions = [] try: list_permissions = "&".join(["=".join(("permissions", p)) for p in self.bad_permissions]) url = f"https://www.googleapis.com/storage/v1/b/{bucket_name}/iam/testPermissions?" + list_permissions - response = self.helpers.request(url) + response = await self.helpers.request(url) permissions = response.json() if isinstance(permissions, dict): bad_permissions = list(permissions.get("permissions", {})) @@ -46,8 +46,8 @@ def check_bucket_open(self, bucket_name, url): msg = f"Open permissions on storage bucket ({perms_str})" return (msg, set()) - def check_bucket_exists(self, bucket_name, url): - response = self.helpers.request(url) + async def check_bucket_exists(self, bucket_name, url): + response = await self.helpers.request(url) status_code = getattr(response, "status_code", 0) existent_bucket = status_code not in (0, 400, 404) - return existent_bucket, set() + return existent_bucket, set(), bucket_name, url diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 6bf98a8604..25cef3d625 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -336,10 +336,11 @@ async def handle_event(self, event): if scheme in ("http", "https"): if num_redirects <= self.max_redirects: url_event = self.make_event(location, "URL_UNVERIFIED", event) - # inherit web spider distance from parent (don't increment) - source_web_spider_distance = getattr(event, "web_spider_distance", 0) - url_event.web_spider_distance = source_web_spider_distance - self.emit_event(url_event) + if url_event is not None: + # inherit web spider distance from parent (don't increment) + source_web_spider_distance = getattr(event, "web_spider_distance", 0) + url_event.web_spider_distance = source_web_spider_distance + self.emit_event(url_event) else: self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}") elif scheme: diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 079f8076ed..b11bb60baf 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -10,7 +10,16 @@ class speculate(BaseInternalModule): in situations where e.g. a port scanner isn't enabled """ - watched_events = ["IP_RANGE", "URL", "URL_UNVERIFIED", "DNS_NAME", "IP_ADDRESS", "HTTP_RESPONSE", "STORAGE_BUCKET"] + watched_events = [ + "IP_RANGE", + "URL", + "URL_UNVERIFIED", + "DNS_NAME", + "DNS_NAME_UNRESOLVED", + "IP_ADDRESS", + "HTTP_RESPONSE", + "STORAGE_BUCKET", + ] produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING"] flags = ["passive"] meta = {"description": "Derive certain event types from others by common sense"} @@ -104,6 +113,9 @@ async def handle_event(self, event): quick=True, ) + # storage buckets etc. + self.helpers.cloud.speculate(event) + async def filter_event(self, event): # don't accept IP_RANGE --> IP_ADDRESS events from self if str(event.module) == "speculate": diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index 9e8282702f..e7f2507b6f 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -145,7 +145,6 @@ async def masscan(self, targets, result_callback, exclude=None, ping=False): stats_file = self.helpers.tempfile_tail(callback=self.verbose) try: with open(stats_file, "w") as stats_fh: - self.critical(f"masscan: {command}") async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh): self.process_output(line, result_callback=result_callback) finally: diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 592ec1ff63..1844e67006 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -11,7 +11,7 @@ class asn(BaseReportModule): # because sometimes IP addresses are re-emitted with lower scope distances accept_dupes = True - def setup(self): + async def setup(self): self.asn_counts = {} self.asn_cache = {} self.sources = ["bgpview", "ripe"] @@ -24,15 +24,15 @@ def setup(self): } return True - def filter_event(self, event): + async def filter_event(self, event): if getattr(event.host, "is_private", False): return False return True - def handle_event(self, event): + async def handle_event(self, event): host = event.host if self.cache_get(host) == False: - asns = list(self.get_asn(host)) + asns = await self.get_asn(host) if not asns: self.cache_put(self.unknown_asn) else: @@ -81,7 +81,7 @@ def cache_get(self, ip): continue return ret - def get_asn(self, ip, retries=1): + async def get_asn(self, ip, retries=1): """ Takes in an IP returns a list of ASNs, e.g.: @@ -90,7 +90,7 @@ def get_asn(self, ip, retries=1): for attempt in range(retries + 1): for i, source in enumerate(list(self.sources)): get_asn_fn = getattr(self, f"get_asn_{source}") - res = get_asn_fn(ip) + res = await get_asn_fn(ip) if res == False: # demote the current source to lowest priority since it just failed self.sources.append(self.sources.pop(i)) @@ -100,9 +100,9 @@ def get_asn(self, ip, retries=1): self.warning(f"Error retrieving ASN via for {ip}") return [] - def get_asn_ripe(self, ip): + async def get_asn_ripe(self, ip): url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}" - response = self.get_url(url, "ASN") + response = await self.get_url(url, "ASN") asns = [] if response == False: return False @@ -116,19 +116,19 @@ def get_asn_ripe(self, ip): if not asn_numbers: asn_numbers = [] for number in asn_numbers: - asn = self.get_asn_metadata_ripe(number) + asn = await self.get_asn_metadata_ripe(number) asn["subnet"] = prefix asns.append(asn) return asns - def get_asn_metadata_ripe(self, asn_number): + async def get_asn_metadata_ripe(self, asn_number): metadata_keys = { "name": ["ASName", "OrgId"], "description": ["OrgName", "OrgTechName", "RTechName"], "country": ["Country"], } url = f"https://stat.ripe.net/data/whois/data.json?resource={asn_number}" - response = self.get_url(url, "ASN Metadata", cache=True) + response = await self.get_url(url, "ASN Metadata", cache=True) if response == False: return False data = response.get("data", {}) @@ -155,9 +155,9 @@ def get_asn_metadata_ripe(self, asn_number): asn["asn"] = str(asn_number) return asn - def get_asn_bgpview(self, ip): + async def get_asn_bgpview(self, ip): url = f"https://api.bgpview.io/ip/{ip}" - data = self.get_url(url, "ASN") + data = await self.get_url(url, "ASN") asns = [] asns_tried = set() if data == False: @@ -175,7 +175,7 @@ def get_asn_bgpview(self, ip): country = details.get("country_code") or prefix.get("country_code") or "" emails = [] if not asn in asns_tried: - emails = self.get_emails_bgpview(asn) + emails = await self.get_emails_bgpview(asn) if emails == False: return False asns_tried.add(asn) @@ -186,10 +186,10 @@ def get_asn_bgpview(self, ip): self.debug(f'No results for "{ip}"') return asns - def get_emails_bgpview(self, asn): + async def get_emails_bgpview(self, asn): contacts = [] url = f"https://api.bgpview.io/asn/{asn}" - data = self.get_url(url, "ASN metadata", cache=True) + data = await self.get_url(url, "ASN metadata", cache=True) if data == False: return False data = data.get("data", {}) @@ -201,11 +201,11 @@ def get_emails_bgpview(self, asn): contacts = [l.strip().lower() for l in email_contacts + abuse_contacts] return list(set(contacts)) - def get_url(self, url, data_type, cache=False): + async def get_url(self, url, data_type, cache=False): kwargs = {} if cache: kwargs["cache_for"] = 60 * 60 * 24 - r = self.helpers.request(url, **kwargs) + r = await self.helpers.request(url, **kwargs) data = {} try: j = r.json() diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index 29a0f0c1f2..788e9d6254 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -19,11 +19,11 @@ class wafw00f(BaseModule): in_scope_only = True - def setup(self): + async def setup(self): self.scanned_hosts = set() return True - def handle_event(self, event): + async def handle_event(self, event): parsed_host = event.parsed host = f"{parsed_host.scheme}://{parsed_host.netloc}/" host_hash = hash(host) @@ -33,14 +33,15 @@ def handle_event(self, event): else: self.scanned_hosts.add(host_hash) - WW = wafw00f_main.WAFW00F(host) - waf_detections = WW.identwaf() + WW = await self.scan.run_in_executor(wafw00f_main.WAFW00F, host) + waf_detections = await self.scan.run_in_executor(WW.identwaf) if waf_detections: - for waf in WW.identwaf(): + for waf in waf_detections: self.emit_event({"host": host, "WAF": waf}, "WAF", source=event) else: if self.config.get("generic_detect") == True: - if WW.genericdetect(): + generic = await self.scan.run_in_executor(WW.genericdetect) + if generic: self.emit_event( {"host": host, "WAF": "generic detection", "info": WW.knowledge["generic"]["reason"]}, "WAF", diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c43f702a15..d66330819d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -641,7 +641,6 @@ def _load_modules(self, modules): async def _status_ticker(self, interval=15): async with self.acatch(): - # while not self.stopped: while 1: await asyncio.sleep(interval) await self.manager.modules_status(_log=True) @@ -672,6 +671,9 @@ async def acatch(self, context="scan", finally_callback=None): except BaseException as e: self._handle_exception(e, context=context) + def run_in_executor(self, *args, **kwargs): + return self._loop.run_in_executor(None, *args, **kwargs) + def _handle_exception(self, e, context="scan", finally_callback=None): if callable(context): context = f"{context.__qualname__}()" diff --git a/bbot/test/helpers.py b/bbot/test/helpers.py index fd457795b8..0cfafafc99 100644 --- a/bbot/test/helpers.py +++ b/bbot/test/helpers.py @@ -18,7 +18,8 @@ def __init__(self, request, **kwargs): self.bbot_scanner = request.getfixturevalue("bbot_scanner") self.config = OmegaConf.merge(self.bbot_config, OmegaConf.create(self.config_overrides)) modules = [self.name] + self.additional_modules - self.scan = self.bbot_scanner( + self.scans = [] + self.scan = self.add_scan( *self.targets, modules=modules, name=f"{self.name}_test", @@ -27,19 +28,25 @@ def __init__(self, request, **kwargs): blacklist=self.blacklist, ) - def patch_scan(self, scan): - return + def add_scan(self, *args, **kwargs): + scan = self.bbot_scanner(*args, **kwargs) + self.scans.append(scan) + return scan - def setup(self): + def setup(self, scan): pass async def run(self): - await self.scan.prep() - self.setup() - self.patch_scan(self.scan) + for i, scan in enumerate(self.scans): + if i == 0: + self.scan = scan + await scan.prep() + self.setup(scan) + self._after_scan_prep() - events = [e async for e in self.scan.start()] - self.check_events(events) + for i, scan in enumerate(self.scans): + events = [e async for e in scan.start()] + self.check_events(events) @abstractmethod def check_events(self, events): diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index f763318bca..11a99fe914 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -79,7 +79,7 @@ class Excavate(HttpxMockHelper): config_overrides = {"web_spider_distance": 1, "web_spider_depth": 1} - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 404 def mock_args(self): @@ -172,7 +172,7 @@ class Excavate_relativelinks(HttpxMockHelper): targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/"] config_overrides = {"web_spider_distance": 1, "web_spider_depth": 1} - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 404 def mock_args(self): @@ -284,7 +284,7 @@ def check_events(self, events): class Anubisdb(RequestMockHelper): - def setup(self): + def setup(self, scan): self.module.abort_if = lambda e: False def mock_args(self): @@ -512,7 +512,7 @@ class Paramminer_headers(HttpxMockHelper): config_overrides = {"modules": {"paramminer_headers": {"wordlist": tempwordlist(["junkword1", "tracestate"])}}} - def setup(self): + def setup(self, scan): from bbot.core.helpers import helper self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" @@ -559,7 +559,7 @@ class Paramminer_getparams(HttpxMockHelper): config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist(["canary", "id"])}}} - def setup(self): + def setup(self, scan): from bbot.core.helpers import helper self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" @@ -606,7 +606,7 @@ class Paramminer_cookies(HttpxMockHelper): config_overrides = {"modules": {"paramminer_cookies": {"wordlist": tempwordlist(["junkcookie", "admincookie"])}}} - def setup(self): + def setup(self, scan): from bbot.core.helpers import helper self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" @@ -738,28 +738,20 @@ class Masscan(MockHelper): ports = range = 9.8.7.6""" - def _after_scan_prep(self): - self.scan.modules["masscan"].masscan_config = self.masscan_config - - async def setup_scan_2(): - config2 = OmegaConf.merge(self.config, OmegaConf.create(self.config_overrides_2)) - self.scan2 = self.bbot_scanner( - *self.targets, - modules=[self.name] + self.additional_modules, - name=f"{self.name}_test", - config=config2, - whitelist=self.whitelist, - blacklist=self.blacklist, - ) - self.patch_scan(self.scan2) - await self.scan2.prep() - self.scan2.modules["masscan"].masscan_config = self.masscan_config - - self.setup_scan_2 = setup_scan_2 + def __init__(self, request): + super().__init__(request) + config2 = OmegaConf.merge(self.config, OmegaConf.create(self.config_overrides_2)) + self.add_scan( + *self.targets, + modules=[self.name] + self.additional_modules, + name=f"{self.name}_test", + config=config2, + whitelist=self.whitelist, + blacklist=self.blacklist, + ) self.masscan_run = False async def run_masscan(self, command, *args, **kwargs): - log.critical(f"patched: {command}") if "masscan" in command[:2]: for l in self.masscan_output.splitlines(): yield l @@ -768,26 +760,30 @@ async def run_masscan(self, command, *args, **kwargs): async for l in self.scan.helpers.run_live(command, *args, **kwargs): yield l - def patch_scan(self, scan): + def setup(self, scan): scan.helpers.run_live = self.run_masscan + scan.modules["masscan"].masscan_config = self.masscan_config async def run(self): - await super().run() - assert self.masscan_run == True, "masscan didn't run when it was supposed to" - await self.setup_scan_2() - self.masscan_run = False - events = [e async for e in self.scan2.start()] - self.check_events(events) - assert self.masscan_run == False, "masscan ran when it wasn't supposed to" + for i, scan in enumerate(self.scans): + await scan.prep() + self.setup(scan) + events = [e async for e in scan.start()] + self.check_events(events) + if i == 0: + assert self.masscan_run == True, "masscan didn't run when it was supposed to" + self.masscan_run = False + else: + assert self.masscan_run == False, "masscan ran when it wasn't supposed to" def check_events(self, events): assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" -class Buckets(HttpxMockHelper, RequestMockHelper): +class Buckets(HttpxMockHelper): providers = ["aws", "gcp", "azure", "digitalocean", "firebase"] - # providers = ["aws"] + # providers = ["azure"] additional_modules = ["excavate", "speculate", "httpx"] + [f"bucket_{p}" for p in providers] config_overrides = { "modules": { @@ -817,7 +813,11 @@ class Buckets(HttpxMockHelper, RequestMockHelper): ] }""" - def patch_scan(self, scan): + def __init__(self, request, **kwargs): + self.httpx_mock = request.getfixturevalue("httpx_mock") + super().__init__(request, **kwargs) + + def setup(self, scan): scan.helpers.word_cloud.mutations = lambda b, cloud=False: [ (b, "dev"), ] @@ -839,56 +839,43 @@ def mock_args(self): """ self.set_expect_requests(expect_args=expect_args, respond_args={"response_data": body}) - def mock_args_requests(self): - self.m.register_uri("GET", requests_mock.ANY, text="", status_code=404) - self.register_uri( - f"https://{self.random_bucket_name_2}.s3-ap-southeast-2.amazonaws.com/", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}.s3-ap-southeast-2.amazonaws.com", text=self.open_aws_bucket, ) - self.register_uri( - f"https://{self.random_bucket_name_2}.fra1.digitaloceanspaces.com/", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}.fra1.digitaloceanspaces.com", text=self.open_digitalocean_bucket, ) - self.register_uri( - f"https://{self.random_bucket_name_2}.blob.core.windows.net/{self.random_bucket_name_2}?restype=container", - text="", - ) - self.register_uri( - f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}/iam/testPermissions?permissions=storage.buckets.setIamPolicy&permissions=storage.objects.list&permissions=storage.objects.get&permissions=storage.objects.create", + self.httpx_mock.add_response( + url=f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}/iam/testPermissions?permissions=storage.buckets.setIamPolicy&permissions=storage.objects.list&permissions=storage.objects.get&permissions=storage.objects.create", text=self.open_gcp_bucket, ) - self.register_uri( - f"https://{self.random_bucket_name_2}.firebaseio.com/.json", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}.firebaseio.com/.json", text="", ) - - self.register_uri( - f"https://{self.random_bucket_name_2}-dev.s3.amazonaws.com/", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}-dev.s3.amazonaws.com", text="", ) - self.register_uri( - f"https://{self.random_bucket_name_2}-dev.fra1.digitaloceanspaces.com/", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}-dev.fra1.digitaloceanspaces.com", text="", ) - self.register_uri( - f"https://{self.random_bucket_name_2}-dev.blob.core.windows.net/{self.random_bucket_name_2}-dev?restype=container", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}-dev.blob.core.windows.net/{self.random_bucket_name_2}-dev?restype=container", text="", ) - self.register_uri( - f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}-dev", + self.httpx_mock.add_response( + url=f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}-dev", text="", ) - self.register_uri( - f"https://{self.random_bucket_name_2}-dev.firebaseio.com/.json", + self.httpx_mock.add_response( + url=f"https://{self.random_bucket_name_2}-dev.firebaseio.com/.json", text="", ) - - def run(self): - with requests_mock.Mocker() as m: - self.m = m - self.mock_args_requests() - events = list(self.scan.start()) - self.check_events(events) + self.httpx_mock.add_response(url=re.compile(".*"), text="", status_code=404) def check_events(self, events): for provider in self.providers: @@ -906,6 +893,7 @@ def check_events(self, events): url = e.data.get("url", "") assert self.random_bucket_name_2 in url assert not self.random_bucket_name_1 in url + assert not f"{self.random_bucket_name_2}-dev" in url # make sure bucket mutations were found assert any( e.type == "STORAGE_BUCKET" @@ -915,7 +903,7 @@ def check_events(self, events): ), f'bucket (dev mutation) not found for provider "{provider}"' -class ASN(RequestMockHelper): +class ASN(HttpxMockHelper): targets = ["8.8.8.8"] response_get_asn_ripe = { "messages": [], @@ -1115,39 +1103,41 @@ class ASN(RequestMockHelper): } config_overrides = {"scope_report_distance": 2} - def __init__(self, config, bbot_scanner, *args): - super().__init__(config, bbot_scanner, *args) - self.scan2 = bbot_scanner( + def __init__(self, request, **kwargs): + super().__init__(request, **kwargs) + self.scan2 = self.add_scan( *self.targets, modules=[self.name] + self.additional_modules, name=f"{self.name}_test_2", config=self.config, ) - self.scan2.prep() - self.module2 = self.scan2.modules[self.name] def mock_args(self): - pass + self.httpx_mock.add_response( + url="https://stat.ripe.net/data/network-info/data.json?resource=8.8.8.8", + text=json.dumps(self.response_get_asn_ripe), + ) + self.httpx_mock.add_response( + url="https://stat.ripe.net/data/whois/data.json?resource=15169", + text=json.dumps(self.response_get_asn_metadata_ripe), + ) + self.httpx_mock.add_response( + url="https://api.bgpview.io/ip/8.8.8.8", text=json.dumps(self.response_get_asn_bgpview) + ) + self.httpx_mock.add_response( + url="https://api.bgpview.io/asn/15169", text=json.dumps(self.response_get_emails_bgpview) + ) - def run(self): - with requests_mock.Mocker() as m: - self.m = m - self.register_uri( - "https://stat.ripe.net/data/network-info/data.json?resource=8.8.8.8", - text=json.dumps(self.response_get_asn_ripe), - ) - self.register_uri( - "https://stat.ripe.net/data/whois/data.json?resource=15169", - text=json.dumps(self.response_get_asn_metadata_ripe), - ) - self.register_uri("https://api.bgpview.io/ip/8.8.8.8", text=json.dumps(self.response_get_asn_bgpview)) - self.register_uri("https://api.bgpview.io/asn/15169", text=json.dumps(self.response_get_emails_bgpview)) - self.module.sources = ["bgpview", "ripe"] - events = list(e for e in self.scan.start() if e.module == self.module) - assert self.check_events(events) - self.module2.sources = ["ripe", "bgpview"] - events2 = list(e for e in self.scan2.start() if e.module == self.module2) - assert self.check_events(events2) + async def run(self): + await self.scan.prep() + self.module.sources = ["bgpview", "ripe"] + events = [e async for e in self.scan.start() if e.module == self.module] + assert self.check_events(events) + await self.scan2.prep() + self.module2 = self.scan2.modules["asn"] + self.module2.sources = ["ripe", "bgpview"] + events2 = [e async for e in self.scan2.start() if e.module == self.module2] + assert self.check_events(events2) def check_events(self, events): asn = False @@ -1169,11 +1159,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "WAF": - if "LiteSpeed" in e.data["WAF"]: - return True - return False + assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events) class Ffuf(HttpxMockHelper): @@ -1310,7 +1296,7 @@ class Ffuf_shortnames(HttpxMockHelper): } } - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 404 seed_events = [] @@ -1509,7 +1495,7 @@ class Iis_shortnames(HttpxMockHelper): config_overrides = {"modules": {"iis_shortnames": {"detect_only": False}}} - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 404 def mock_args(self): @@ -1792,7 +1778,7 @@ class Bypass403(HttpxMockHelper): targets = ["http://127.0.0.1:8888/test"] - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 403 def mock_args(self): @@ -1812,7 +1798,7 @@ class Bypass403_aspnetcookieless(HttpxMockHelper): targets = ["http://127.0.0.1:8888/admin.aspx"] - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 403 def mock_args(self): @@ -1832,7 +1818,7 @@ class Bypass403_waf(HttpxMockHelper): targets = ["http://127.0.0.1:8888/test"] - def setup(self): + def setup(self, scan): self.bbot_httpserver.no_handler_status_code = 403 def mock_args(self): From 0a37a8e0095d0e0b353c57726333334e32bfb2fb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 12 May 2023 16:59:48 -0400 Subject: [PATCH 014/387] better error handling in speculate --- bbot/modules/internal/speculate.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index b11bb60baf..1e00d1bdb7 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -90,10 +90,11 @@ async def handle_event(self, event): url_parents = self.helpers.url_parents(event.data) for up in url_parents: url_event = self.make_event(f"{up}/", "URL_UNVERIFIED", source=event) - # inherit web spider distance from parent (don't increment) - source_web_spider_distance = getattr(event, "web_spider_distance", 0) - url_event.web_spider_distance = source_web_spider_distance - self.emit_event(url_event) + if url_event is not None: + # inherit web spider distance from parent (don't increment) + source_web_spider_distance = getattr(event, "web_spider_distance", 0) + url_event.web_spider_distance = source_web_spider_distance + self.emit_event(url_event) # from hosts if emit_open_ports: From 6286cc30b941ee5c85f0c6f526fe5a06eaa6b9d8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 12 May 2023 16:59:54 -0400 Subject: [PATCH 015/387] better error handling in speculate --- bbot/core/helpers/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 606cca4d3f..cd351b8dc9 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -69,7 +69,7 @@ def AsyncClient(self, *args, **kwargs): async def request(self, *args, **kwargs): raise_error = kwargs.pop("raise_error", False) # TODO: use this - cache_for = kwargs.pop("cache_for", None) # noqa + cache_for = kwargs.pop("cache_for", None) # noqa # in case of URL only, assume GET request if len(args) == 1: From 65e8263f5b136dd7b589d54ff07daa123ae5d761 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 12 May 2023 17:07:39 -0400 Subject: [PATCH 016/387] update readme --- README.md | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9b9c96e3eb..9b9e05f446 100644 --- a/README.md +++ b/README.md @@ -149,17 +149,26 @@ bbot -f subdomain-enum -t evilcorp.com --output-modules neo4j # Usage ~~~ $ bbot --help -usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-n SCAN_NAME] [-m MODULE [MODULE ...]] [-l] [-em MODULE [MODULE ...]] - [-f FLAG [FLAG ...]] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [-o DIR] [-c [CONFIG ...]] [--allow-deadly] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config] - [--save-wordcloud FILE] [--load-wordcloud FILE] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] [--version] +usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-m MODULE [MODULE ...]] [-l] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] + [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] [-o DIR] [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config] [--save-wordcloud FILE] [--load-wordcloud FILE] + [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] [--version] Bighuge BLS OSINT Tool options: -h, --help show this help message and exit --help-all Display full help including module config options - -n SCAN_NAME, --name SCAN_NAME - Name of scan (default: random) + +Target: + -t TARGET [TARGET ...], --targets TARGET [TARGET ...] + Targets to seed the scan + -w WHITELIST [WHITELIST ...], --whitelist WHITELIST [WHITELIST ...] + What's considered in-scope (by default it's the same as --targets) + -b BLACKLIST [BLACKLIST ...], --blacklist BLACKLIST [BLACKLIST ...] + Don't touch these things + --strict-scope Don't consider subdomains of target/whitelist to be in-scope + +Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] Modules to enable. Choices: affiliates,anubisdb,asn,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_firebase,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,crobat,crt,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,fingerprintx,fullhunt,generic_ssrf,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ipneighbor,ipstack,leakix,masscan,massdns,naabu,ntlm,nuclei,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,skymem,smuggler,social,sslcert,subdomain_hijack,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye -l, --list-modules List available modules. @@ -168,15 +177,19 @@ options: -f FLAG [FLAG ...], --flags FLAG [FLAG ...] Enable modules by flag. Choices: active,affiliates,aggressive,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] - Disable modules that don't have these flags (e.g. -rf passive) + Only enable modules with these flags (e.g. -rf passive) -ef FLAG [FLAG ...], --exclude-flags FLAG [FLAG ...] Disable modules with these flags. (e.g. -ef aggressive) -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] Output module(s). Choices: asset_inventory,csv,http,human,json,neo4j,python,web_report,websocket + --allow-deadly Enable the use of highly aggressive modules + +Scan: + -n SCAN_NAME, --name SCAN_NAME + Name of scan (default: random) -o DIR, --output-dir DIR -c [CONFIG ...], --config [CONFIG ...] custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234' - --allow-deadly Enable the use of highly aggressive modules -v, --verbose Be more verbose -d, --debug Enable debugging -s, --silent Be quiet @@ -185,15 +198,6 @@ options: --dry-run Abort before executing scan --current-config Show current config in YAML format -Target: - -t TARGET [TARGET ...], --targets TARGET [TARGET ...] - Targets to seed the scan - -w WHITELIST [WHITELIST ...], --whitelist WHITELIST [WHITELIST ...] - What's considered in-scope (by default it's the same as --targets) - -b BLACKLIST [BLACKLIST ...], --blacklist BLACKLIST [BLACKLIST ...] - Don't touch these things - --strict-scope Don't consider subdomains of target/whitelist to be in-scope - Word cloud: Save/load wordlist of common words gathered during a scan From 245e6a425281949911b0feac638989203d1e88c4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 13 May 2023 21:41:28 -0400 Subject: [PATCH 017/387] all module tests passing --- bbot/core/helpers/misc.py | 8 +- bbot/core/helpers/web.py | 5 + bbot/modules/bucket_gcp.py | 2 +- bbot/modules/bypass403.py | 6 +- bbot/modules/deadly/ffuf.py | 33 +++-- bbot/modules/deadly/nuclei.py | 22 ++-- bbot/modules/deadly/vhost.py | 57 +++------ bbot/modules/ffuf_shortnames.py | 32 ++--- bbot/modules/hunt.py | 2 +- bbot/modules/iis_shortnames.py | 54 ++++---- bbot/modules/social.py | 4 +- bbot/modules/url_manipulation.py | 6 +- bbot/scanner/manager.py | 1 + bbot/test/modules_test_classes.py | 142 +++++++-------------- bbot/test/test_step_1/test_modules_full.py | 40 +++--- bbot/test/test_step_2/test_threadpool.py | 17 --- 16 files changed, 180 insertions(+), 251 deletions(-) delete mode 100644 bbot/test/test_step_2/test_threadpool.py diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index c2b39214a7..7a6391a768 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -45,6 +45,7 @@ def is_domain(d): "evilcorp.co.uk" --> True "www.evilcorp.co.uk" --> False """ + d, _ = split_host_port(d) extracted = tldextract(d) if extracted.domain and not extracted.subdomain: return True @@ -56,6 +57,7 @@ def is_subdomain(d): "www.evilcorp.co.uk" --> True "evilcorp.co.uk" --> False """ + d, _ = split_host_port(d) extracted = tldextract(d) if extracted.domain and extracted.subdomain: return True @@ -126,8 +128,9 @@ def parent_domain(d): "www.evilcorp.co.uk" --> "evilcorp.co.uk" "evilcorp.co.uk" --> "evilcorp.co.uk" """ + host, port = split_host_port(d) if is_subdomain(d): - return ".".join(str(d).split(".")[1:]) + return make_netloc(".".join(str(host).split(".")[1:]), port) return d @@ -534,10 +537,13 @@ def gen_numbers(n, padding=2): def make_netloc(host, port): """ + ("192.168.1.1", None) --> "192.168.1.1" ("192.168.1.1", 443) --> "192.168.1.1:443" ("evilcorp.com", 80) --> "evilcorp.com:80" ("dead::beef", 443) --> "[dead::beef]:443" """ + if port is None: + return host if is_ip(host, version=6): host = f"[{host}]" return f"{host}:{port}" diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index cd351b8dc9..6ed9fe8427 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -71,6 +71,11 @@ async def request(self, *args, **kwargs): # TODO: use this cache_for = kwargs.pop("cache_for", None) # noqa + # allow vs follow, httpx why?? + allow_redirects = kwargs.pop("allow_redirects", None) + if allow_redirects is not None and "follow_redirects" not in kwargs: + kwargs["follow_redirects"] = allow_redirects + # in case of URL only, assume GET request if len(args) == 1: kwargs["url"] = args[0] diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py index 06cc588946..b7e96d5b1d 100644 --- a/bbot/modules/bucket_gcp.py +++ b/bbot/modules/bucket_gcp.py @@ -39,7 +39,7 @@ async def check_bucket_open(self, bucket_name, url): if isinstance(permissions, dict): bad_permissions = list(permissions.get("permissions", {})) except Exception as e: - self.warning(f'Failed to enumerate permissions for bucket "{bucket_name}": {e}') + self.info(f'Failed to enumerate permissions for bucket "{bucket_name}": {e}') msg = "" if bad_permissions: perms_str = ",".join(bad_permissions) diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py index b1205cec33..d798c03441 100644 --- a/bbot/modules/bypass403.py +++ b/bbot/modules/bypass403.py @@ -81,7 +81,7 @@ class bypass403(BaseModule): meta = {"description": "Check 403 pages for common bypasses"} in_scope_only = True - def handle_event(self, event): + async def handle_event(self, event): try: compare_helper = self.helpers.http_compare(event.data, allow_redirects=True) except HttpCompareError as e: @@ -94,7 +94,7 @@ def handle_event(self, event): headers = dict(sig[2]) else: headers = None - match, reasons, reflection, subject_response = compare_helper.compare( + match, reasons, reflection, subject_response = await compare_helper.compare( sig[1], headers=headers, method=sig[0], allow_redirects=True ) @@ -121,7 +121,7 @@ def handle_event(self, event): else: self.debug(f"Status code changed to {str(subject_response.status_code)}, ignoring") - def filter_event(self, event): + async def filter_event(self, event): if ("status-403" in event.tags) or ("status-401" in event.tags): return True return False diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index a034261490..709b1d1be6 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -46,20 +46,18 @@ class ffuf(BaseModule): in_scope_only = True - def setup(self): + async def setup(self): self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) wordlist_url = self.config.get("wordlist", "") self.debug(f"Using wordlist [{wordlist_url}]") - self.wordlist = self.helpers.wordlist(wordlist_url) - f = open(self.wordlist, "r") - self.wordlist_lines = f.readlines() - f.close() + self.wordlist = await self.helpers.wordlist(wordlist_url) + self.wordlist_lines = list(self.helpers.read_file(self.wordlist)) self.tempfile, tempfile_len = self.generate_templist() self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]") - self.extensions = self.config.get("extensions") + self.extensions = self.config.get("extensions", "") return True - def handle_event(self, event): + async def handle_event(self, event): if self.helpers.url_depth(event.data) > self.config.get("max_depth"): self.debug(f"Exceeded max depth, aborting event") return @@ -77,17 +75,17 @@ def handle_event(self, event): for ext in self.extensions.split(","): exts.append(f".{ext}") - filters = self.baseline_ffuf(fixed_url, exts=exts) - for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters): + filters = await self.baseline_ffuf(fixed_url, exts=exts) + async for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters): self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) - def filter_event(self, event): + async def filter_event(self, event): if "endpoint" in event.tags: self.debug(f"rejecting URL [{event.data}] because we don't ffuf endpoints") return False return True - def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal"): + async def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal"): filters = {} for ext in exts: self.debug(f"running baseline for URL [{url}] with ext [{ext}]") @@ -102,7 +100,7 @@ def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal"): canary_length += 2 canary_temp_file = self.helpers.tempfile(canary_list, pipe=False) - for canary_r in self.execute_ffuf( + async for canary_r in self.execute_ffuf( canary_temp_file, url, prefix=prefix, @@ -194,7 +192,7 @@ def baseline_ffuf(self, url, exts=[""], prefix="", suffix="", mode="normal"): return filters - def execute_ffuf( + async def execute_ffuf( self, tempfile, url, @@ -261,7 +259,7 @@ def execute_ffuf( command.append("-mc") command.append("all") - for found in self.helpers.run_live(command): + async for found in self.helpers.run_live(command): try: found_json = json.loads(found) input_json = found_json.get("input", {}) @@ -280,8 +278,9 @@ def execute_ffuf( if mode == "normal": # before emitting, we are going to send another baseline. This will immediately catch things like a WAF flipping blocking on us mid-scan if baseline == False: - pre_emit_temp_canary = list( - self.execute_ffuf( + pre_emit_temp_canary = [ + f + async for f in self.execute_ffuf( self.helpers.tempfile( ["".join(random.choice(string.ascii_lowercase) for i in range(4))], pipe=False, @@ -294,7 +293,7 @@ def execute_ffuf( baseline=True, filters=filters, ) - ) + ] if len(pre_emit_temp_canary) == 0: yield found_json else: diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 9a36387162..f7c7ee2c86 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -13,7 +13,7 @@ class nuclei(BaseModule): batch_size = 25 options = { - "version": "2.8.9", + "version": "2.9.4", "tags": "", "templates": "", "severity": "", @@ -50,11 +50,11 @@ class nuclei(BaseModule): deps_pip = ["pyyaml~=6.0"] in_scope_only = True - def setup(self): + async def setup(self): # attempt to update nuclei templates self.nuclei_templates_dir = self.helpers.tools_dir / "nuclei-templates" self.info("Updating Nuclei templates") - update_results = self.helpers.run( + update_results = await self.helpers.run( ["nuclei", "-update-template-dir", self.nuclei_templates_dir, "-update-templates"] ) if update_results.stderr: @@ -127,10 +127,10 @@ def setup(self): return True - def handle_batch(self, *events): + async def handle_batch(self, *events): temp_target = self.helpers.make_target(events) nuclei_input = [str(e.data) for e in events] - for severity, template, host, url, name, extracted_results in self.execute_nuclei(nuclei_input): + async for severity, template, host, url, name, extracted_results in self.execute_nuclei(nuclei_input): # this is necessary because sometimes nuclei is inconsistent about the data returned in the host field cleaned_host = temp_target.get(host) source_event = self.correlate_event(events, cleaned_host) @@ -170,10 +170,10 @@ def correlate_event(self, events, host): return event self.warning("Failed to correlate nuclei result with event") - def execute_nuclei(self, nuclei_input): + async def execute_nuclei(self, nuclei_input): command = [ "nuclei", - "-json", + "-jsonl", "-update-template-dir", self.nuclei_templates_dir, "-rate-limit", @@ -207,7 +207,7 @@ def execute_nuclei(self, nuclei_input): stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status) try: with open(stats_file, "w") as stats_fh: - for line in self.helpers.run_live(command, input=nuclei_input, stderr=stats_fh): + async for line in self.helpers.run_live(command, input=nuclei_input, stderr=stats_fh): try: j = json.loads(line) except json.decoder.JSONDecodeError: @@ -258,11 +258,11 @@ def log_nuclei_status(self, line): status = f"[{duration}] | Templates: {templates} | Hosts: {hosts} | RPS: {rps} | Matched: {matched} | Errors: {errors} | Requests: {requests}/{total} ({percent}%)" self.info(status) - def cleanup(self): + async def cleanup(self): resume_file = self.helpers.current_dir / "resume.cfg" resume_file.unlink(missing_ok=True) - def filter_event(self, event): + async def filter_event(self, event): if self.config.get("directory_only", True): if "endpoint" in event.tags: self.debug( @@ -299,7 +299,7 @@ def find_budget_paths(self, budget): def get_yaml_request_attr(self, yf, attr): p = self.parse_yaml(yf) - requests = p.get("requests", []) + requests = p.get("http", []) for r in requests: raw = r.get("raw") if not raw: diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index 3c0a5c27a8..f4675e10fe 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -1,9 +1,7 @@ -from bbot.modules.deadly.ffuf import ffuf - -from urllib.parse import urlparse -import random -import string import base64 +from urllib.parse import urlparse + +from bbot.modules.deadly.ffuf import ffuf class vhost(ffuf): @@ -38,29 +36,12 @@ class vhost(ffuf): in_scope_only = True - def setup(self): - self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) + async def setup(self): self.scanned_hosts = {} self.wordcloud_tried_hosts = set() - self.wordlist = self.helpers.wordlist(self.config.get("wordlist")) - f = open(self.wordlist, "r") - self.wordlist_lines = f.readlines() - f.close() - self.ignore_redirects = True - self.tempfile, tempfile_len = self.generate_templist() - return True - - @staticmethod - def get_parent_domain(domain): - domain_parts = domain.split(".") - - if len(domain_parts) >= 3: - parent_domain = ".".join(domain_parts[1:]) - return parent_domain - else: - return domain - - def handle_event(self, event): + return await super().setup() + + async def handle_event(self, event): if not self.helpers.is_ip(event.host) or self.config.get("force_basehost"): host = f"{event.parsed.scheme}://{event.parsed.netloc}" if host in self.scanned_hosts.keys(): @@ -73,24 +54,24 @@ def handle_event(self, event): if self.config.get("force_basehost"): basehost = self.config.get("force_basehost") else: - basehost = self.get_parent_domain(event.parsed.netloc) + basehost = self.helpers.parent_domain(event.parsed.netloc) self.debug(f"Using basehost: {basehost}") - for vhost in self.ffuf_vhost(host, f".{basehost}", event): + async for vhost in self.ffuf_vhost(host, f".{basehost}", event): self.verbose(f"Starting mutations check for {vhost}") - for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=self.mutations_check(vhost)): + async for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=self.mutations_check(vhost)): pass # check existing host for mutations self.verbose("Checking for vhost mutations on main host") - for vhost in self.ffuf_vhost( + async for vhost in self.ffuf_vhost( host, f".{basehost}", event, wordlist=self.mutations_check(event.parsed.netloc.split(".")[0]) ): pass # special vhost list self.verbose("Checking special vhost list") - for vhost in self.ffuf_vhost( + async for vhost in self.ffuf_vhost( host, "", event, @@ -99,13 +80,15 @@ def handle_event(self, event): ): pass - def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=False): - filters = self.baseline_ffuf(f"{host}/", exts=[""], suffix=basehost, mode="hostheader") + async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=False): + filters = await self.baseline_ffuf(f"{host}/", exts=[""], suffix=basehost, mode="hostheader") self.debug(f"Baseline completed and returned these filters:") self.debug(filters) if not wordlist: wordlist = self.tempfile - for r in self.execute_ffuf(wordlist, host, exts=[""], suffix=basehost, filters=filters, mode="hostheader"): + async for r in self.execute_ffuf( + wordlist, host, exts=[""], suffix=basehost, filters=filters, mode="hostheader" + ): found_vhost_b64 = r["input"]["FUZZ"] vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()} if f"{vhost_dict['vhost']}{basehost}" != event.parsed.netloc: @@ -123,7 +106,7 @@ def mutations_check(self, vhost): mutations_list_file = self.helpers.tempfile(mutations_list, pipe=False) return mutations_list_file - def finish(self): + async def finish(self): # check existing hosts with wordcloud tempfile = self.helpers.tempfile(list(self.helpers.word_cloud.keys()), pipe=False) @@ -135,9 +118,9 @@ def finish(self): if self.config.get("force_basehost"): basehost = self.config.get("force_basehost") else: - basehost = self.get_parent_domain(event.parsed.netloc) + basehost = self.helpers.parent_domain(event.parsed.netloc) - for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=tempfile): + async for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=tempfile): pass self.wordcloud_tried_hosts.add(host) diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index 513f61267b..d181319f03 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -72,22 +72,20 @@ class ffuf_shortnames(ffuf): in_scope_only = True - def setup(self): + async def setup(self): self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) wordlist = self.config.get("wordlist", "") if not wordlist: wordlist = f"{self.helpers.wordlist_dir}/ffuf_shortname_candidates.txt" self.debug(f"Using [{wordlist}] for shortname candidate list") - self.wordlist = self.helpers.wordlist(wordlist) - f = open(self.wordlist, "r") - self.wordlist_lines = f.readlines() - f.close() + self.wordlist = await self.helpers.wordlist(wordlist) + self.wordlist_lines = list(self.helpers.read_file(self.wordlist)) wordlist_extensions = self.config.get("wordlist_extensions", "") if not wordlist_extensions: wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt" self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list") - self.wordlist_extensions = self.helpers.wordlist(wordlist_extensions) + self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions) self.extensions = self.config.get("extensions") self.ignore_redirects = self.config.get("ignore_redirects") @@ -116,10 +114,10 @@ def find_delimeter(self, hint): return d, hint.split(d)[0], hint.split(d)[1] return None - def filter_event(self, event): + async def filter_event(self, event): return True - def handle_event(self, event): + async def handle_event(self, event): if event.source.type == "URL": filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower() @@ -151,11 +149,11 @@ def handle_event(self, event): if tempfile_len > 0: if "shortname-file" in event.tags: for ext in used_extensions: - for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): + async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) elif "shortname-directory" in event.tags: - for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): + async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): r_url = f"{r['url'].rstrip('/')}/" self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) @@ -166,7 +164,9 @@ def handle_event(self, event): delimeter, prefix, partial_hint = delimeter_r self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]") tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) - for r in self.execute_ffuf(tempfile, root_url, prefix=f"{prefix}{delimeter}", exts=["/"]): + async for r in self.execute_ffuf( + tempfile, root_url, prefix=f"{prefix}{delimeter}", exts=["/"] + ): self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) elif "shortname-file" in event.tags: @@ -176,14 +176,14 @@ def handle_event(self, event): delimeter, prefix, partial_hint = delimeter_r self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]") tempfile, tempfile_len = self.generate_templist(prefix=partial_hint) - for r in self.execute_ffuf( + async for r in self.execute_ffuf( tempfile, root_url, prefix=f"{prefix}{delimeter}", suffix=f".{ext}" ): self.emit_event( r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] ) - def finish(self): + async def finish(self): if self.config.get("find_common_prefixes"): per_host_collection = dict(self.per_host_collection) self.per_host_collection.clear() @@ -208,7 +208,7 @@ def finish(self): f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix} and partial_hint: {partial_hint}" ) - for r in self.execute_ffuf(tempfile, url, prefix=prefix, exts=["/"]): + async for r in self.execute_ffuf(tempfile, url, prefix=prefix, exts=["/"]): self.emit_event( r["url"], "URL_UNVERIFIED", @@ -222,7 +222,9 @@ def finish(self): self.verbose( f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix}, extension: .{ext}, and partial_hint: {partial_hint}" ) - for r in self.execute_ffuf(tempfile, url, prefix=prefix, suffix=f".{ext}"): + async for r in self.execute_ffuf( + tempfile, url, prefix=prefix, suffix=f".{ext}" + ): self.emit_event( r["url"], "URL_UNVERIFIED", diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index 8f61a5a2af..64abd181eb 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -326,7 +326,7 @@ def extract_params(self, body): self.debug(f"FOUND PARAM ({s}) IN A TAG GET PARAMS") yield s - def handle_event(self, event): + async def handle_event(self, event): body = event.data.get("body", "") for p in self.extract_params(body): for k in hunt_param_dict.keys(): diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 779781f52a..3ee204321f 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -24,7 +24,7 @@ class iis_shortnames(BaseModule): max_event_handlers = 8 - def detect(self, target): + async def detect(self, target): technique = None detections = [] random_string = self.helpers.rand_string(8) @@ -32,8 +32,8 @@ def detect(self, target): test_url = f"{target}*~1*/a.aspx" for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]: - control = self.helpers.request(method=method, url=control_url, allow_redirects=False, timeout=10) - test = self.helpers.request(method=method, url=test_url, allow_redirects=False, timeout=10) + control = await self.helpers.request(method=method, url=control_url, allow_redirects=False, timeout=10) + test = await self.helpers.request(method=method, url=test_url, allow_redirects=False, timeout=10) if (control != None) and (test != None): if control.status_code != test.status_code: technique = f"{str(control.status_code)}/{str(test.status_code)} HTTP Code" @@ -46,7 +46,7 @@ def detect(self, target): technique = "HTTP Body Error Message" return detections - def setup(self): + async def setup(self): self.scanned_tracker_lock = Lock() self.scanned_tracker = set() return True @@ -55,10 +55,10 @@ def setup(self): def normalize_url(url): return str(url.rstrip("/") + "/").lower() - def directory_confirm(self, target, method, url_hint, affirmative_status_code): + async def directory_confirm(self, target, method, url_hint, affirmative_status_code): payload = encode_all(f"{url_hint}") url = f"{target}{payload}" - directory_confirm_result = self.helpers.request( + directory_confirm_result = await self.helpers.request( method=method, url=url, allow_redirects=False, retries=2, timeout=10 ) @@ -67,7 +67,7 @@ def directory_confirm(self, target, method, url_hint, affirmative_status_code): else: return False - def duplicate_check(self, target, method, url_hint, affirmative_status_code): + async def duplicate_check(self, target, method, url_hint, affirmative_status_code): duplicates = [] count = 2 base_hint = re.sub(r"~\d", "", url_hint) @@ -77,7 +77,7 @@ def duplicate_check(self, target, method, url_hint, affirmative_status_code): payload = encode_all(f"{base_hint}~{str(count)}*") url = f"{target}{payload}{suffix}" - duplicate_check_results = self.helpers.request( + duplicate_check_results = await self.helpers.request( method=method, url=url, allow_redirects=False, retries=2, timeout=10 ) if duplicate_check_results.status_code != affirmative_status_code: @@ -92,30 +92,30 @@ def duplicate_check(self, target, method, url_hint, affirmative_status_code): return duplicates - def threaded_request(self, method, url, affirmative_status_code): - r = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) + async def threaded_request(self, method, url, affirmative_status_code, c): + r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) if r is not None: if r.status_code == affirmative_status_code: - return True + return True, c + return None, c - def solve_shortname_recursive( + async def solve_shortname_recursive( self, method, target, prefix, affirmative_status_code, extension_mode=False, node_count=0 ): url_hint_list = [] found_results = False - futures = {} + tasks = [] for c in valid_chars: suffix = "\\a.aspx" wildcard = "*" if extension_mode else "*~1*" payload = encode_all(f"{prefix}{c}{wildcard}") url = f"{target}{payload}{suffix}" - future = self.submit_task(self.threaded_request, method, url, affirmative_status_code) - futures[future] = c + task = self.helpers.create_task(self.threaded_request(method, url, affirmative_status_code, c)) + tasks.append(task) - for future in self.helpers.as_completed(futures): - c = futures[future] - result = future.result() + for task in self.helpers.as_completed(tasks): + result, c = await task if result: found_results = True node_count += 1 @@ -130,12 +130,12 @@ def solve_shortname_recursive( wildcard = "~1*" payload = encode_all(f"{prefix}{c}{wildcard}") url = f"{target}{payload}{suffix}" - r = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) + r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10) if r is not None: if r.status_code == affirmative_status_code: url_hint_list.append(f"{prefix}{c}") - url_hint_list += self.solve_shortname_recursive( + url_hint_list += await self.solve_shortname_recursive( method, target, f"{prefix}{c}", affirmative_status_code, extension_mode, node_count=node_count ) if len(prefix) > 0 and found_results == False: @@ -143,12 +143,12 @@ def solve_shortname_recursive( self.verbose(f"Found new (possibly partial) URL_HINT: {prefix} from node {target}") return url_hint_list - def handle_event(self, event): + async def handle_event(self, event): normalized_url = self.normalize_url(event.data) with self.scanned_tracker_lock: self.scanned_tracker.add(normalized_url) - detections = self.detect(normalized_url) + detections = await self.detect(normalized_url) technique_strings = [] if detections: @@ -171,7 +171,7 @@ def handle_event(self, event): break file_name_hints = list( - set(self.solve_shortname_recursive(method, normalized_url, "", affirmative_status_code)) + set(await self.solve_shortname_recursive(method, normalized_url, "", affirmative_status_code)) ) if len(file_name_hints) == 0: continue @@ -184,19 +184,19 @@ def handle_event(self, event): file_name_hints_dedupe = file_name_hints[:] for x in file_name_hints_dedupe: - duplicates = self.duplicate_check(normalized_url, method, x, affirmative_status_code) + duplicates = await self.duplicate_check(normalized_url, method, x, affirmative_status_code) if duplicates: file_name_hints += duplicates # check for the case of a folder and file with the same filename for d in file_name_hints: - if self.directory_confirm(normalized_url, method, d, affirmative_status_code): + if await self.directory_confirm(normalized_url, method, d, affirmative_status_code): self.verbose(f"Confirmed Directory URL_HINT: {d} from node {normalized_url}") url_hint_list.append(d) for y in file_name_hints: - file_name_extension_hints = self.solve_shortname_recursive( + file_name_extension_hints = await self.solve_shortname_recursive( method, normalized_url, f"{y}.", affirmative_status_code, extension_mode=True ) for z in file_name_extension_hints: @@ -212,7 +212,7 @@ def handle_event(self, event): hint_type = "shortname-directory" self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type]) - def filter_event(self, event): + async def filter_event(self, event): if "dir" in event.tags: with self.scanned_tracker_lock: if self.normalize_url(event.data) not in self.scanned_tracker: diff --git a/bbot/modules/social.py b/bbot/modules/social.py index ed890daf2e..14af427d1b 100644 --- a/bbot/modules/social.py +++ b/bbot/modules/social.py @@ -22,11 +22,11 @@ class social(BaseModule): scope_distance_modifier = 1 - def setup(self): + async def setup(self): self.compiled_regexes = {k: re.compile(v) for k, v in self.social_media_regex.items()} return True - def handle_event(self, event): + async def handle_event(self, event): for platform, regex in self.compiled_regexes.items(): for match in regex.findall(event.data): social_media_links = {"platform": platform, "url": match} diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index 91eb4c5c50..851761dc2a 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -14,7 +14,7 @@ class url_manipulation(BaseModule): "allow_redirects": "Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default." } - def setup(self): + async def setup(self): # ([string]method,[string]path,[bool]strip trailing slash) self.signatures = [] @@ -38,7 +38,7 @@ def setup(self): self.allow_redirects = self.config.get("allow_redirects", True) return True - def handle_event(self, event): + async def handle_event(self, event): try: compare_helper = self.helpers.http_compare( event.data, allow_redirects=self.allow_redirects, include_cache_buster=False @@ -53,7 +53,7 @@ def handle_event(self, event): for sig in self.signatures: sig = self.format_signature(sig, event) - match, reasons, reflection, subject_response = compare_helper.compare( + match, reasons, reflection, subject_response = await compare_helper.compare( sig[1], method=sig[0], allow_redirects=self.allow_redirects ) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 377b690f32..7538bb89c2 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -39,6 +39,7 @@ async def init_events(self): for event in sorted_events: self.scan.verbose(f"Target: {event}") self.queue_event(event) + await asyncio.sleep(0.1) self.scan._finished_init = True async def emit_event(self, event, *args, **kwargs): diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index 11a99fe914..30088c65cf 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -1184,11 +1184,8 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "URL_UNVERIFIED": - if "admin" in e.data: - return True - return False + assert any(e.type == "URL_UNVERIFIED" and "admin" in e.data for e in events) + assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) class Ffuf_extensions(HttpxMockHelper): @@ -1207,11 +1204,8 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "URL_UNVERIFIED": - if "console" in e.data: - return True - return False + assert any(e.type == "URL_UNVERIFIED" and "console" in e.data for e in events) + assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) class Vhost(HttpxMockHelper): @@ -1273,15 +1267,11 @@ def check_events(self, events): if e.data["vhost"] == "secret": wordcloud_detection = True - if ( - basic_detection - and mutaton_of_detected - and basehost_mutation - and special_vhost_list - and wordcloud_detection - ): - return True - return False + assert basic_detection + assert mutaton_of_detected + assert basehost_mutation + assert special_vhost_list + assert wordcloud_detection class Ffuf_shortnames(HttpxMockHelper): @@ -1477,17 +1467,13 @@ def check_events(self, events): if e.data == "http://127.0.0.1:8888/short.pl": short_extensions_detection = True - if ( - basic_detection - and directory_detection - and prefix_detection - and delimeter_detection - and directory_delimeter_detection - and prefix_delimeter_detection - and short_extensions_detection - ): - return True - return False + assert basic_detection + assert directory_detection + assert prefix_detection + assert delimeter_detection + assert directory_delimeter_detection + assert prefix_delimeter_detection + assert short_extensions_detection class Iis_shortnames(HttpxMockHelper): @@ -1540,9 +1526,8 @@ def check_events(self, events): if e.type == "URL_HINT" and e.data == "http://127.0.0.1:8888/BLSHAX~1": url_hintEmitted = True - if vulnerabilityEmitted and url_hintEmitted: - return True - return False + assert vulnerabilityEmitted + assert url_hintEmitted class Nuclei_manual(HttpxMockHelper): @@ -1568,10 +1553,11 @@ class Nuclei_manual(HttpxMockHelper): "web_spider_depth": 1, "modules": { "nuclei": { + "version": "2.9.4", "mode": "manual", "concurrency": 2, "ratelimit": 10, - "templates": "/tmp/.bbot_test/tools/nuclei-templates/miscellaneous/", + "templates": "/tmp/.bbot_test/tools/nuclei-templates/http/miscellaneous/", "interactsh_disable": True, "directory_only": False, } @@ -1596,9 +1582,8 @@ def check_events(self, events): first_run_detect = True elif "Copyright" in e.data["description"]: second_run_detect = True - if first_run_detect and second_run_detect: - return True - return False + assert first_run_detect + assert second_run_detect class Nuclei_severe(HttpxMockHelper): @@ -1621,11 +1606,10 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "VULNERABILITY": - if "Generic Linux - Local File Inclusion" in e.data["description"]: - return True - return False + assert any( + e.type == "VULNERABILITY" and "Generic Linux - Local File Inclusion" in e.data["description"] + for e in events + ) class Nuclei_technology(HttpxMockHelper): @@ -1636,9 +1620,9 @@ class Nuclei_technology(HttpxMockHelper): "modules": {"nuclei": {"mode": "technology", "concurrency": 2, "tags": "apache"}}, } - def __init__(self, config, bbot_scanner, bbot_httpserver, caplog, *args, **kwargs): + def __init__(self, request, caplog, **kwargs): self.caplog = caplog - super().__init__(config, bbot_scanner, bbot_httpserver, *args, **kwargs) + super().__init__(request, **kwargs) def mock_args(self): expect_args = {"method": "GET", "uri": "/"} @@ -1651,12 +1635,7 @@ def mock_args(self): def check_events(self, events): if "Using Interactsh Server" in self.caplog.text: return False - - for e in events: - if e.type == "FINDING": - if "apache" in e.data["description"]: - return True - return False + assert any(e.type == "FINDING" and "apache" in e.data["description"] for e in events) class Nuclei_budget(HttpxMockHelper): @@ -1680,11 +1659,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING": - if "SpiderFoot" in e.data["description"]: - return True - return False + assert any(e.type == "FINDING" and "SpiderFoot" in e.data["description"] for e in events) class Url_manipulation(HttpxMockHelper): @@ -1716,14 +1691,12 @@ def mock_args(self): self.set_expect_requests(respond_args=respond_args) def check_events(self, events): - for e in events: - if ( - e.type == "FINDING" - and e.data["description"] - == f"Url Manipulation: [body] Sig: [Modified URL: http://127.0.0.1:8888/?{self.module.rand_string}=.xml]" - ): - return True - return False + assert any( + e.type == "FINDING" + and e.data["description"] + == f"Url Manipulation: [body] Sig: [Modified URL: http://127.0.0.1:8888/?{self.module.rand_string}=.xml]" + for e in events + ) class Naabu(HttpxMockHelper): @@ -1733,10 +1706,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "OPEN_TCP_PORT": - return True - return False + assert any(e.type == "OPEN_TCP_PORT" for e in events) class Social(HttpxMockHelper): @@ -1748,11 +1718,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "SOCIAL": - if e.data["platform"] == "discord": - return True - return False + assert any(e.type == "SOCIAL" and e.data["platform"] == "discord" for e in events) class Hunt(HttpxMockHelper): @@ -1764,13 +1730,10 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if ( - e.type == "FINDING" - and e.data["description"] == "Found potential INSECURE CRYPTOGRAPHY parameter [cipher]" - ): - return True - return False + assert any( + e.type == "FINDING" and e.data["description"] == "Found potential INSECURE CRYPTOGRAPHY parameter [cipher]" + for e in events + ) class Bypass403(HttpxMockHelper): @@ -1787,10 +1750,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING": - return True - return False + assert any(e.type == "FINDING" for e in events) class Bypass403_aspnetcookieless(HttpxMockHelper): @@ -1807,10 +1767,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING": - return True - return False + assert any(e.type == "FINDING" for e in events) class Bypass403_waf(HttpxMockHelper): @@ -1827,10 +1784,7 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "FINDING": - return False - return True + assert not any(e.type == "FINDING" for e in events) class Speculate_subdirectories(HttpxMockHelper): @@ -1851,8 +1805,4 @@ def mock_args(self): self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check_events(self, events): - for e in events: - if e.type == "URL_UNVERIFIED": - if e.data == "http://127.0.0.1:8888/subdir1/": - return True - return False + assert any(e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/subdir1/" for e in events) diff --git a/bbot/test/test_step_1/test_modules_full.py b/bbot/test/test_step_1/test_modules_full.py index 3dea25d310..5c6ec38651 100644 --- a/bbot/test/test_step_1/test_modules_full.py +++ b/bbot/test/test_step_1/test_modules_full.py @@ -145,44 +145,50 @@ async def test_ffuf_extensions(request): @pytest.mark.asyncio -async def test_bypass403(request): - x = Bypass403(request) +async def test_vhost(request): + x = Vhost(request) await x.run() @pytest.mark.asyncio -async def test_bypass403_waf(request): - x = Bypass403_waf(request, module_name="bypass403") +async def test_ffuf_shortnames(request): + x = Ffuf_shortnames(request) await x.run() @pytest.mark.asyncio -async def test_bypass403_aspnetcookieless(request): - x = Bypass403_aspnetcookieless(request, module_name="bypass403") +async def test_iis_shortnames(request): + x = Iis_shortnames(request) await x.run() @pytest.mark.asyncio -async def test_ffuf_shortnames(request): - x = Ffuf_shortnames(request) +async def test_nuclei_manual(request): + x = Nuclei_manual(request, module_name="nuclei") await x.run() @pytest.mark.asyncio -async def test_iis_shortnames(request): - x = Iis_shortnames(request) +async def test_bypass403(request): + x = Bypass403(request) await x.run() @pytest.mark.asyncio -async def test_nuclei_technology(request, caplog): - x = Nuclei_technology(request, caplog, module_name="nuclei") +async def test_bypass403_waf(request): + x = Bypass403_waf(request, module_name="bypass403") await x.run() @pytest.mark.asyncio -async def test_nuclei_manual(request): - x = Nuclei_manual(request, module_name="nuclei") +async def test_bypass403_aspnetcookieless(request): + x = Bypass403_aspnetcookieless(request, module_name="bypass403") + await x.run() + + +@pytest.mark.asyncio +async def test_nuclei_technology(request, caplog): + x = Nuclei_technology(request, caplog, module_name="nuclei") await x.run() @@ -216,12 +222,6 @@ async def test_hunt(request): await x.run() -@pytest.mark.asyncio -async def test_vhost(request): - x = Vhost(request) - await x.run() - - @pytest.mark.asyncio async def test_speculate_subdirectories(request): x = Speculate_subdirectories(request, module_name="speculate") diff --git a/bbot/test/test_step_2/test_threadpool.py b/bbot/test/test_step_2/test_threadpool.py deleted file mode 100644 index d4238b0062..0000000000 --- a/bbot/test/test_step_2/test_threadpool.py +++ /dev/null @@ -1,17 +0,0 @@ -def test_threadpool(): - from bbot.core.helpers.threadpool import BBOTThreadPoolExecutor, ThreadPoolWrapper, NamedLock, as_completed - - with BBOTThreadPoolExecutor(max_workers=3) as executor: - pool = ThreadPoolWrapper(executor) - add_one = lambda x: x + 1 - futures = [pool.submit_task(add_one, y) for y in [0, 1, 2, 3, 4]] - results = [] - for f in as_completed(futures): - results.append(f.result()) - assert tuple(sorted(results)) == (1, 2, 3, 4, 5) - - nl = NamedLock(max_size=5) - for i in range(50): - nl.get_lock(str(i)) - assert len(nl._cache) == 5 - assert tuple(nl._cache.keys()) == tuple(hash(str(x)) for x in [45, 46, 47, 48, 49]) From 530c5e20b123c907d9d18b9da934c5e0aa1bb20e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 13 May 2023 22:01:54 -0400 Subject: [PATCH 018/387] fixing more tests --- bbot/core/helpers/misc.py | 1 + bbot/test/test_step_2/test_helpers.py | 38 +- bbot/test/test_step_2/test_modules_basic.py | 451 ++++++++++---------- bbot/test/test_step_2/test_web.py | 14 + 4 files changed, 246 insertions(+), 258 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 7a6391a768..fda9cc8f2a 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -125,6 +125,7 @@ def split_host_port(d): def parent_domain(d): """ "www.internal.evilcorp.co.uk" --> "internal.evilcorp.co.uk" + "www.internal.evilcorp.co.uk:8080" --> "internal.evilcorp.co.uk:8080" "www.evilcorp.co.uk" --> "evilcorp.co.uk" "evilcorp.co.uk" --> "evilcorp.co.uk" """ diff --git a/bbot/test/test_step_2/test_helpers.py b/bbot/test/test_step_2/test_helpers.py index e3f9bc1d3b..2d9c6809df 100644 --- a/bbot/test/test_step_2/test_helpers.py +++ b/bbot/test/test_step_2/test_helpers.py @@ -1,12 +1,12 @@ import re import datetime import ipaddress -import requests_mock from ..bbot_fixtures import * -def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): +@pytest.mark.asyncio +async def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): ### URL ### bad_urls = ( "http://e.co/index.html", @@ -46,17 +46,6 @@ def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): assert helpers.url_depth("http://evilcorp.com/") == 0 assert helpers.url_depth("http://evilcorp.com") == 0 - ### HTTP COMPARE ### - with requests_mock.Mocker() as m: - m.get(re.compile(r"http://www.example.com.*"), text="wat") - compare_helper = helpers.http_compare("http://www.example.com") - compare_helper.compare("http://www.example.com", headers={"asdf": "asdf"}) - compare_helper.compare("http://www.example.com", cookies={"asdf": "asdf"}) - compare_helper.compare("http://www.example.com", check_reflection=True) - compare_helper.compare_body({"asdf": "fdsa"}, {"fdsa": "asdf"}) - for mode in ("getparam", "header", "cookie"): - compare_helper.canary_check("http://www.example.com", mode=mode) == True - ### MISC ### assert helpers.is_domain("evilcorp.co.uk") assert not helpers.is_domain("www.evilcorp.co.uk") @@ -382,12 +371,12 @@ def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): for i in range(100): f.write(f"{i}\n") assert len(list(open(test_file).readlines())) == 100 - assert helpers.wordlist(test_file).is_file() - truncated_file = helpers.wordlist(test_file, lines=10) + assert (await helpers.wordlist(test_file)).is_file() + truncated_file = await helpers.wordlist(test_file, lines=10) assert truncated_file.is_file() assert len(list(open(truncated_file).readlines())) == 10 with pytest.raises(WordlistError): - helpers.wordlist("/tmp/a9pseoysadf/asdkgjaosidf") + await helpers.wordlist("/tmp/a9pseoysadf/asdkgjaosidf") test_file.unlink() # misc DNS helpers @@ -406,21 +395,6 @@ def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): with pytest.raises(NTLMError): helpers.ntlm.ntlmdecode("asdf") - # interact.sh - with requests_mock.Mocker() as m: - from bbot.core.helpers.interactsh import server_list - - for server in server_list: - m.post(re.compile(rf"https://{server}/.*"), text="nope") - - interactsh_client = helpers.interactsh() - with pytest.raises(InteractshError): - interactsh_client.register() - with pytest.raises(InteractshError): - list(interactsh_client.poll()) - with pytest.raises(InteractshError): - interactsh_client.deregister() - test_filesize = Path("/tmp/test_filesize") test_filesize.touch() assert test_filesize.is_file() @@ -435,7 +409,7 @@ def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver): assert helpers.human_to_bytes("428.24GB") == 459819198709 scan1 = bbot_scanner(modules="ipneighbor") - scan1.load_modules() + await scan1.load_modules() assert int(helpers.get_size(scan1.modules["ipneighbor"])) > 0 diff --git a/bbot/test/test_step_2/test_modules_basic.py b/bbot/test/test_step_2/test_modules_basic.py index 7ab5de917b..301ca2ce16 100644 --- a/bbot/test/test_step_2/test_modules_basic.py +++ b/bbot/test/test_step_2/test_modules_basic.py @@ -1,242 +1,241 @@ import re -import requests_mock from contextlib import suppress from ..bbot_fixtures import * -def test_modules_basic(patch_commands, patch_ansible, scan, helpers, events, bbot_config, bbot_scanner): +@pytest.mark.asyncio +async def test_modules_basic(patch_commands, patch_ansible, scan, helpers, events, bbot_config, bbot_scanner, httpx_mock): fallback_nameservers = scan.helpers.temp_dir / "nameservers.txt" with open(fallback_nameservers, "w") as f: f.write("8.8.8.8\n") - with requests_mock.Mocker() as m: - for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"): - m.request(http_method, re.compile(r".*"), text='{"test": "test"}') - - # event filtering - from bbot.modules.base import BaseModule - from bbot.modules.output.base import BaseOutputModule - from bbot.modules.report.base import BaseReportModule - from bbot.modules.internal.base import BaseInternalModule - - # output module specific event filtering tests - base_output_module = BaseOutputModule(scan) - base_output_module.watched_events = ["IP_ADDRESS"] - localhost = scan.make_event("127.0.0.1", source=scan.root_event) - assert base_output_module._event_precheck(localhost)[0] == True - localhost._internal = True - assert base_output_module._event_precheck(localhost)[0] == False - localhost._force_output = True - assert base_output_module._event_precheck(localhost)[0] == True - localhost._omit = True - assert base_output_module._event_precheck(localhost)[0] == False - - # common event filtering tests - for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule): - base_module = module_class(scan) - localhost2 = scan.make_event("127.0.0.2", source=events.subdomain) - localhost2.make_in_scope() - # base cases - base_module._watched_events = None - base_module.watched_events = ["*"] - assert base_module._event_precheck(events.emoji)[0] == True - base_module._watched_events = None - base_module.watched_events = ["IP_ADDRESS"] - assert base_module._event_precheck(events.ipv4)[0] == True - assert base_module._event_precheck(events.domain)[0] == False - assert base_module._event_precheck(events.localhost)[0] == True - assert base_module._event_precheck(localhost2)[0] == True - # target only - base_module.target_only = True - assert base_module._event_precheck(localhost2)[0] == False - localhost2.add_tag("target") - assert base_module._event_precheck(localhost2)[0] == True - base_module.target_only = False - # special case for IPs and ranges - base_module.watched_events = ["IP_ADDRESS", "IP_RANGE"] - ip_range = scan.make_event("127.0.0.0/24", dummy=True) - localhost4 = scan.make_event("127.0.0.1", source=ip_range) - localhost4.make_in_scope() - localhost4.module = "plumbus" - assert base_module._event_precheck(localhost4)[0] == True - localhost4.module = "speculate" - assert base_module._event_precheck(localhost4)[0] == False - - # in scope only - base_module.in_scope_only = True - localhost3 = scan.make_event("127.0.0.2", source=events.subdomain) - valid, reason = base_module._event_postcheck(localhost3) - if base_module._type == "output": - assert valid - else: - assert not valid - assert reason == "it did not meet in_scope_only filter criteria" - base_module.in_scope_only = False - base_module.scope_distance_modifier = 0 - localhost4 = scan.make_event("127.0.0.1", source=events.subdomain) - valid, reason = base_module._event_postcheck(events.localhost) + for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"): + httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"}) + + # event filtering + from bbot.modules.base import BaseModule + from bbot.modules.output.base import BaseOutputModule + from bbot.modules.report.base import BaseReportModule + from bbot.modules.internal.base import BaseInternalModule + + # output module specific event filtering tests + base_output_module = BaseOutputModule(scan) + base_output_module.watched_events = ["IP_ADDRESS"] + localhost = scan.make_event("127.0.0.1", source=scan.root_event) + assert base_output_module._event_precheck(localhost)[0] == True + localhost._internal = True + assert base_output_module._event_precheck(localhost)[0] == False + localhost._force_output = True + assert base_output_module._event_precheck(localhost)[0] == True + localhost._omit = True + assert base_output_module._event_precheck(localhost)[0] == False + + # common event filtering tests + for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule): + base_module = module_class(scan) + localhost2 = scan.make_event("127.0.0.2", source=events.subdomain) + localhost2.make_in_scope() + # base cases + base_module._watched_events = None + base_module.watched_events = ["*"] + assert base_module._event_precheck(events.emoji)[0] == True + base_module._watched_events = None + base_module.watched_events = ["IP_ADDRESS"] + assert base_module._event_precheck(events.ipv4)[0] == True + assert base_module._event_precheck(events.domain)[0] == False + assert base_module._event_precheck(events.localhost)[0] == True + assert base_module._event_precheck(localhost2)[0] == True + # target only + base_module.target_only = True + assert base_module._event_precheck(localhost2)[0] == False + localhost2.add_tag("target") + assert base_module._event_precheck(localhost2)[0] == True + base_module.target_only = False + # special case for IPs and ranges + base_module.watched_events = ["IP_ADDRESS", "IP_RANGE"] + ip_range = scan.make_event("127.0.0.0/24", dummy=True) + localhost4 = scan.make_event("127.0.0.1", source=ip_range) + localhost4.make_in_scope() + localhost4.module = "plumbus" + assert base_module._event_precheck(localhost4)[0] == True + localhost4.module = "speculate" + assert base_module._event_precheck(localhost4)[0] == False + + # in scope only + base_module.in_scope_only = True + localhost3 = scan.make_event("127.0.0.2", source=events.subdomain) + valid, reason = await base_module._event_postcheck(localhost3) + if base_module._type == "output": assert valid - - base_output_module = BaseOutputModule(scan) - base_output_module.watched_events = ["IP_ADDRESS"] - - scan2 = bbot_scanner( - modules=list(set(available_modules + available_internal_modules)), - output_modules=list(available_output_modules), - config=bbot_config, - ) - scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers - patch_commands(scan2) - patch_ansible(scan2) - scan2.load_modules() - scan2.status = "RUNNING" - - # attributes, descriptions, etc. - for module_name, module in scan2.modules.items(): - # flags - assert module._type in ("internal", "output", "scan") - - # module preloading - all_preloaded = module_loader.preloaded() - assert "massdns" in all_preloaded - assert "DNS_NAME" in all_preloaded["massdns"]["watched_events"] - assert "DNS_NAME" in all_preloaded["massdns"]["produced_events"] - assert "subdomain-enum" in all_preloaded["massdns"]["flags"] - assert "wordlist" in all_preloaded["massdns"]["config"] - assert type(all_preloaded["massdns"]["config"]["max_resolvers"]) == int - assert all_preloaded["sslcert"]["deps"]["pip"] - assert all_preloaded["sslcert"]["deps"]["apt"] - assert all_preloaded["massdns"]["deps"]["ansible"] - - for module_name, preloaded in all_preloaded.items(): - # either active or passive and never both - flags = preloaded.get("flags", []) - if preloaded["type"] == "scan": - assert ("active" in flags and not "passive" in flags) or ( - not "active" in flags and "passive" in flags - ), f'module "{module_name}" must have either "active" or "passive" flag' - assert preloaded.get("meta", {}).get("description", ""), f"{module_name} must have a description" - - # attribute checks - watched_events = preloaded.get("watched_events") - produced_events = preloaded.get("produced_events") - - assert type(watched_events) == list - assert type(produced_events) == list - if not preloaded.get("type", "") in ("internal",): - assert watched_events, f"{module_name}.watched_events must not be empty" - assert type(watched_events) == list, f"{module_name}.watched_events must be of type list" - assert type(produced_events) == list, f"{module_name}.produced_events must be of type list" - assert all( - [type(t) == str for t in watched_events] - ), f"{module_name}.watched_events entries must be of type string" - assert all( - [type(t) == str for t in produced_events] - ), f"{module_name}.produced_events entries must be of type string" - - assert type(preloaded.get("deps_pip", [])) == list, f"{module_name}.deps_pip must be of type list" - assert ( - type(preloaded.get("deps_pip_constraints", [])) == list - ), f"{module_name}.deps_pip_constraints must be of type list" - assert type(preloaded.get("deps_apt", [])) == list, f"{module_name}.deps_apt must be of type list" - assert type(preloaded.get("deps_shell", [])) == list, f"{module_name}.deps_shell must be of type list" - assert type(preloaded.get("config", None)) == dict, f"{module_name}.options must be of type list" + else: + assert not valid + assert reason == "it did not meet in_scope_only filter criteria" + base_module.in_scope_only = False + base_module.scope_distance_modifier = 0 + localhost4 = scan.make_event("127.0.0.1", source=events.subdomain) + valid, reason = await base_module._event_postcheck(events.localhost) + assert valid + + base_output_module = BaseOutputModule(scan) + base_output_module.watched_events = ["IP_ADDRESS"] + + scan2 = bbot_scanner( + modules=list(set(available_modules + available_internal_modules)), + output_modules=list(available_output_modules), + config=bbot_config, + ) + scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers + patch_commands(scan2) + patch_ansible(scan2) + scan2.load_modules() + scan2.status = "RUNNING" + + # attributes, descriptions, etc. + for module_name, module in scan2.modules.items(): + # flags + assert module._type in ("internal", "output", "scan") + + # module preloading + all_preloaded = module_loader.preloaded() + assert "massdns" in all_preloaded + assert "DNS_NAME" in all_preloaded["massdns"]["watched_events"] + assert "DNS_NAME" in all_preloaded["massdns"]["produced_events"] + assert "subdomain-enum" in all_preloaded["massdns"]["flags"] + assert "wordlist" in all_preloaded["massdns"]["config"] + assert type(all_preloaded["massdns"]["config"]["max_resolvers"]) == int + assert all_preloaded["sslcert"]["deps"]["pip"] + assert all_preloaded["sslcert"]["deps"]["apt"] + assert all_preloaded["massdns"]["deps"]["ansible"] + + for module_name, preloaded in all_preloaded.items(): + # either active or passive and never both + flags = preloaded.get("flags", []) + if preloaded["type"] == "scan": + assert ("active" in flags and not "passive" in flags) or ( + not "active" in flags and "passive" in flags + ), f'module "{module_name}" must have either "active" or "passive" flag' + assert preloaded.get("meta", {}).get("description", ""), f"{module_name} must have a description" + + # attribute checks + watched_events = preloaded.get("watched_events") + produced_events = preloaded.get("produced_events") + + assert type(watched_events) == list + assert type(produced_events) == list + if not preloaded.get("type", "") in ("internal",): + assert watched_events, f"{module_name}.watched_events must not be empty" + assert type(watched_events) == list, f"{module_name}.watched_events must be of type list" + assert type(produced_events) == list, f"{module_name}.produced_events must be of type list" + assert all( + [type(t) == str for t in watched_events] + ), f"{module_name}.watched_events entries must be of type string" + assert all( + [type(t) == str for t in produced_events] + ), f"{module_name}.produced_events entries must be of type string" + + assert type(preloaded.get("deps_pip", [])) == list, f"{module_name}.deps_pip must be of type list" + assert ( + type(preloaded.get("deps_pip_constraints", [])) == list + ), f"{module_name}.deps_pip_constraints must be of type list" + assert type(preloaded.get("deps_apt", [])) == list, f"{module_name}.deps_apt must be of type list" + assert type(preloaded.get("deps_shell", [])) == list, f"{module_name}.deps_shell must be of type list" + assert type(preloaded.get("config", None)) == dict, f"{module_name}.options must be of type list" + assert ( + type(preloaded.get("options_desc", None)) == dict + ), f"{module_name}.options_desc must be of type list" + # options must have descriptions + assert set(preloaded.get("config", {})) == set( + preloaded.get("options_desc", {}) + ), f"{module_name}.options do not match options_desc" + # descriptions most not be blank + assert all( + o for o in preloaded.get("options_desc", {}).values() + ), f"{module_name}.options_desc descriptions must not be blank" + + # setups + futures = {} + for module_name, module in scan2.modules.items(): + log.info(f"Testing {module_name}.setup()") + future = scan2._thread_pool.submit(module.setup) + futures[future] = module + for future in helpers.as_completed(futures): + module = futures[future] + result = future.result() + if type(result) == tuple: + assert len(result) == 2, f"if tuple, {module.name}.setup() return value must have length of 2" + status, msg = result + assert status in ( + True, + False, + None, + ), f"if tuple, the first element of {module.name}.setup()'s return value must be either True, False, or None" assert ( - type(preloaded.get("options_desc", None)) == dict - ), f"{module_name}.options_desc must be of type list" - # options must have descriptions - assert set(preloaded.get("config", {})) == set( - preloaded.get("options_desc", {}) - ), f"{module_name}.options do not match options_desc" - # descriptions most not be blank - assert all( - o for o in preloaded.get("options_desc", {}).values() - ), f"{module_name}.options_desc descriptions must not be blank" - - # setups - futures = {} - for module_name, module in scan2.modules.items(): - log.info(f"Testing {module_name}.setup()") - future = scan2._thread_pool.submit(module.setup) + type(msg) == str + ), f"if tuple, the second element of {module.name}.setup()'s return value must be a message of type str" + else: + assert result in ( + True, + False, + None, + ), f"{module.name}.setup() must return a status of either True, False, or None" + if result == False: + module.set_error_state() + + futures.clear() + + # handle_event / handle_batch + futures = {} + for module_name, module in scan2.modules.items(): + module.emit_event = lambda *args, **kwargs: None + module._filter = lambda *args, **kwargs: True, "" + events_to_submit = [e for e in events.all if e.type in module.watched_events] + if module.batch_size > 1: + log.info(f"Testing {module_name}.handle_batch()") + future = scan2._thread_pool.submit(module.handle_batch, *events_to_submit) futures[future] = module - for future in helpers.as_completed(futures): - module = futures[future] - result = future.result() - if type(result) == tuple: - assert len(result) == 2, f"if tuple, {module.name}.setup() return value must have length of 2" - status, msg = result - assert status in ( - True, - False, - None, - ), f"if tuple, the first element of {module.name}.setup()'s return value must be either True, False, or None" - assert ( - type(msg) == str - ), f"if tuple, the second element of {module.name}.setup()'s return value must be a message of type str" - else: - assert result in ( - True, - False, - None, - ), f"{module.name}.setup() must return a status of either True, False, or None" - if result == False: - module.set_error_state() - - futures.clear() - - # handle_event / handle_batch - futures = {} - for module_name, module in scan2.modules.items(): - module.emit_event = lambda *args, **kwargs: None - module._filter = lambda *args, **kwargs: True, "" - events_to_submit = [e for e in events.all if e.type in module.watched_events] - if module.batch_size > 1: - log.info(f"Testing {module_name}.handle_batch()") - future = scan2._thread_pool.submit(module.handle_batch, *events_to_submit) + else: + for e in events_to_submit: + log.info(f"Testing {module_name}.handle_event()") + future = scan2._thread_pool.submit(module.handle_event, e) futures[future] = module - else: - for e in events_to_submit: - log.info(f"Testing {module_name}.handle_event()") - future = scan2._thread_pool.submit(module.handle_event, e) - futures[future] = module - for future in helpers.as_completed(futures): - try: - assert future.result() == None - except Exception as e: - import traceback - - module = futures[future] - assert module.errored == True, f'Error in module "{module}": {e}\n{traceback.format_exc()}' - futures.clear() - - # finishes - futures = {} - for module_name, module in scan2.modules.items(): - log.info(f"Testing {module_name}.finish()") - future = scan2._thread_pool.submit(module.finish) - futures[future] = module - for future in helpers.as_completed(futures): + for future in helpers.as_completed(futures): + try: assert future.result() == None - futures.clear() + except Exception as e: + import traceback - # cleanups - futures = {} - for module_name, module in scan2.modules.items(): - log.info(f"Testing {module_name}.cleanup()") - future = scan2._thread_pool.submit(module.cleanup) - futures[future] = module - for future in helpers.as_completed(futures): - assert future.result() == None - futures.clear() - - # event filters - for module_name, module in scan2.modules.items(): - log.info(f"Testing {module_name}.filter_event()") - result = module.filter_event(events.emoji) - with suppress(ValueError, TypeError): - result, reason = result - assert result in ( - True, - False, - ), f"{module_name}.filter_event() must return either True or False" + module = futures[future] + assert module.errored == True, f'Error in module "{module}": {e}\n{traceback.format_exc()}' + futures.clear() + + # finishes + futures = {} + for module_name, module in scan2.modules.items(): + log.info(f"Testing {module_name}.finish()") + future = scan2._thread_pool.submit(module.finish) + futures[future] = module + for future in helpers.as_completed(futures): + assert future.result() == None + futures.clear() + + # cleanups + futures = {} + for module_name, module in scan2.modules.items(): + log.info(f"Testing {module_name}.cleanup()") + future = scan2._thread_pool.submit(module.cleanup) + futures[future] = module + for future in helpers.as_completed(futures): + assert future.result() == None + futures.clear() + + # event filters + for module_name, module in scan2.modules.items(): + log.info(f"Testing {module_name}.filter_event()") + result = module.filter_event(events.emoji) + with suppress(ValueError, TypeError): + result, reason = result + assert result in ( + True, + False, + ), f"{module_name}.filter_event() must return either True or False" diff --git a/bbot/test/test_step_2/test_web.py b/bbot/test/test_step_2/test_web.py index 323abf8c6f..92fcfa1afe 100644 --- a/bbot/test/test_step_2/test_web.py +++ b/bbot/test/test_step_2/test_web.py @@ -1,3 +1,5 @@ +import re + from ..bbot_fixtures import * @@ -139,3 +141,15 @@ async def test_web_curl(bbot_scanner, bbot_config, bbot_httpserver): headers_url = bbot_httpserver.url_for("/test-custom-http-headers-curl") curl_result = await helpers.curl(url=headers_url) assert curl_result == "curl_yep_headers" + + +@pytest.mark.asyncio +async def test_web_http_compare(httpx_mock, helpers): + httpx_mock.add_response(re.compile(r"http://www.example.com.*"), text="wat") + compare_helper = helpers.http_compare("http://www.example.com") + await compare_helper.compare("http://www.example.com", headers={"asdf": "asdf"}) + await compare_helper.compare("http://www.example.com", cookies={"asdf": "asdf"}) + await compare_helper.compare("http://www.example.com", check_reflection=True) + compare_helper.compare_body({"asdf": "fdsa"}, {"fdsa": "asdf"}) + for mode in ("getparam", "header", "cookie"): + assert await compare_helper.canary_check("http://www.example.com", mode=mode) == True From 7a737f92fde2afbd5aa50164d01a217e40f2276c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 13 May 2023 22:02:46 -0400 Subject: [PATCH 019/387] blacked --- bbot/test/test_step_2/test_helpers.py | 1 - bbot/test/test_step_2/test_modules_basic.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bbot/test/test_step_2/test_helpers.py b/bbot/test/test_step_2/test_helpers.py index 2d9c6809df..9639f6110c 100644 --- a/bbot/test/test_step_2/test_helpers.py +++ b/bbot/test/test_step_2/test_helpers.py @@ -1,4 +1,3 @@ -import re import datetime import ipaddress diff --git a/bbot/test/test_step_2/test_modules_basic.py b/bbot/test/test_step_2/test_modules_basic.py index 301ca2ce16..5c5285ad2c 100644 --- a/bbot/test/test_step_2/test_modules_basic.py +++ b/bbot/test/test_step_2/test_modules_basic.py @@ -5,7 +5,9 @@ @pytest.mark.asyncio -async def test_modules_basic(patch_commands, patch_ansible, scan, helpers, events, bbot_config, bbot_scanner, httpx_mock): +async def test_modules_basic( + patch_commands, patch_ansible, scan, helpers, events, bbot_config, bbot_scanner, httpx_mock +): fallback_nameservers = scan.helpers.temp_dir / "nameservers.txt" with open(fallback_nameservers, "w") as f: f.write("8.8.8.8\n") @@ -141,9 +143,7 @@ async def test_modules_basic(patch_commands, patch_ansible, scan, helpers, event assert type(preloaded.get("deps_apt", [])) == list, f"{module_name}.deps_apt must be of type list" assert type(preloaded.get("deps_shell", [])) == list, f"{module_name}.deps_shell must be of type list" assert type(preloaded.get("config", None)) == dict, f"{module_name}.options must be of type list" - assert ( - type(preloaded.get("options_desc", None)) == dict - ), f"{module_name}.options_desc must be of type list" + assert type(preloaded.get("options_desc", None)) == dict, f"{module_name}.options_desc must be of type list" # options must have descriptions assert set(preloaded.get("config", {})) == set( preloaded.get("options_desc", {}) From 92b3a886a718aff0b68da4a408733a296c5d478a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 May 2023 13:41:09 -0400 Subject: [PATCH 020/387] agent tests --- bbot/agent/agent.py | 176 ++++++++++++-------- bbot/agent/messages.py | 4 +- bbot/cli.py | 8 +- bbot/core/configurator/environ.py | 17 +- bbot/core/helpers/misc.py | 4 +- bbot/modules/base.py | 1 + bbot/modules/ipneighbor.py | 6 +- bbot/scanner/dispatcher.py | 19 ++- bbot/scanner/scanner.py | 32 ++-- bbot/test/bbot_fixtures.py | 24 +-- bbot/test/test.conf | 2 +- bbot/test/test_step_2/test_agent.py | 149 +++++++++++++++-- bbot/test/test_step_2/test_cloud_helpers.py | 5 +- bbot/test/test_step_2/test_config.py | 5 +- 14 files changed, 313 insertions(+), 139 deletions(-) diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py index 3076119bef..f2a98b0cd1 100644 --- a/bbot/agent/agent.py +++ b/bbot/agent/agent.py @@ -1,15 +1,16 @@ import json +import asyncio import logging -import threading import traceback -import websocket -from time import sleep +import websockets from omegaconf import OmegaConf from . import messages import bbot.core.errors from bbot.scanner import Scanner from bbot.scanner.dispatcher import Dispatcher +from bbot.core.helpers.misc import urlparse, split_host_port +from bbot.core.configurator.environ import prepare_environment log = logging.getLogger("bbot.core.agent") @@ -17,55 +18,97 @@ class Agent: def __init__(self, config): self.config = config + prepare_environment(self.config) self.url = self.config.get("agent_url", "") + self.parsed_url = urlparse(self.url) + self.host, self.port = split_host_port(self.parsed_url.netloc) self.token = self.config.get("agent_token", "") self.scan = None - self.thread = None - self._scan_lock = threading.Lock() + self.task = None + self._ws = None + self._scan_lock = asyncio.Lock() self.dispatcher = Dispatcher() self.dispatcher.on_status = self.on_scan_status self.dispatcher.on_finish = self.on_scan_finish def setup(self): - websocket.enableTrace(False) if not self.url: log.error(f"Must specify agent_url") return False if not self.token: log.error(f"Must specify agent_token") return False - self.ws = websocket.WebSocketApp( - f"{self.url}/control/", - on_open=self.on_open, - on_message=self.on_message, - on_error=self.on_error, - on_close=self.on_close, - header={"Authorization": f"Bearer {self.token}"}, - ) return True - def start(self): - not_keyboardinterrupt = False + async def ws(self, rebuild=False): + if self._ws is None or rebuild: + kwargs = {"close_timeout": 0.5} + if self.token: + kwargs.update({"extra_headers": {"Authorization": f"Bearer {self.token}"}}) + verbs = ("Building", "Built") + if rebuild: + verbs = ("Rebuilding", "Rebuilt") + log.debug(f"{verbs[0]} websocket connection to {self.url}") + self._ws = await websockets.connect(self.url, **kwargs) + log.debug(f"{verbs[1]} websocket connection to {self.url}") + return self._ws + + async def start(self): + rebuild = False while 1: - not_keyboardinterrupt = self.ws.run_forever() - if not not_keyboardinterrupt: - break - sleep(1) + ws = await self.ws(rebuild=rebuild) + rebuild = False + try: + message = await ws.recv() + log.debug(f"Got message: {message}") + try: + message = json.loads(message) + message = messages.Message(**message) + + if message.command == "ping": + if self.scan is None: + await self.send({"conversation": str(message.conversation), "message_type": "pong"}) + + command_type = getattr(messages, message.command, None) + if command_type is None: + log.warning(f'Invalid command: "{message.command}"') + continue + + command_args = command_type(**message.arguments) + command_fn = getattr(self, message.command) + response = await self.err_handle(command_fn, **command_args.dict()) + log.info(str(response)) + await self.send({"conversation": str(message.conversation), "message": response}) + + except json.decoder.JSONDecodeError as e: + log.warning(f'Failed to decode message "{message}": {e}') + log.trace(traceback.format_exc()) + continue + except Exception as e: + log.debug(f"Error receiving message: {e}") + log.debug(traceback.format_exc()) + await asyncio.sleep(1) + rebuild = True - def send(self, message): + async def send(self, message): + rebuild = False while 1: try: - self.ws.send(json.dumps(message)) + ws = await self.ws(rebuild=rebuild) + j = json.dumps(message) + log.debug(f"Sending message of length {len(message)}") + await ws.send(j) + rebuild = False break except Exception as e: - if getattr(self.scan, "stopping", True): - break log.warning(f"Error sending message: {e}, retrying") log.trace(traceback.format_exc()) - sleep(1) + await asyncio.sleep(1) + # rebuild = True - def on_message(self, ws, message): + async def on_message(self, websocket, path): + message = await websocket.recv() try: message = json.loads(message) except Exception as e: @@ -75,8 +118,10 @@ def on_message(self, ws, message): if message.command == "ping": if self.scan is None: - self.send({"conversation": str(message.conversation), "message_type": "pong"}) - return + await self.send({"conversation": str(message.conversation), "message_type": "pong"}) + return + else: + log.warning(f'Invalid command: "{message.command}"') command_type = None try: @@ -86,21 +131,13 @@ def on_message(self, ws, message): command_args = command_type(**message.arguments) command_fn = getattr(self, message.command) - response = self.err_handle(command_fn, **command_args.dict()) - log.info(str(response)) - self.send({"conversation": str(message.conversation), "message": response}) - - def on_error(self, ws, error): - log.warning(f"on_error: {error}") - - def on_close(self, ws, close_status_code, close_msg): - log.warning("Closed connection") + async with self.error_hndle(): + response = await command_fn(**command_args.dict()) + log.info(str(response)) + await self.send({"conversation": str(message.conversation), "message": response}) - def on_open(self, ws): - log.success("Opened connection") - - def start_scan(self, scan_id="", targets=[], modules=[], output_modules=[], config={}): - with self._scan_lock: + async def start_scan(self, scan_id="", name=None, targets=[], modules=[], output_modules=[], config={}): + async with self._scan_lock: if self.scan is None: log.success( f"Starting scan with targets={targets}, modules={modules}, output_modules={output_modules}" @@ -111,27 +148,40 @@ def start_scan(self, scan_id="", targets=[], modules=[], output_modules=[], conf config = OmegaConf.create(config) config = OmegaConf.merge(self.config, config, output_module_config) output_modules = list(set(output_modules + ["websocket"])) - self.scan = Scanner( + scan = Scanner( *targets, scan_id=scan_id, + name=name, modules=modules, output_modules=output_modules, config=config, dispatcher=self.dispatcher, ) - self.thread = threading.Thread(target=self._start_scan, args=(self.scan,), daemon=True) - self.thread.start() + self.task = asyncio.create_task(self._start_scan_task(scan)) - return {"success": f"Started scan", "scan_id": self.scan.id} + return {"success": f"Started scan", "scan_id": scan.id} else: msg = f"Scan {self.scan.id} already in progress" log.warning(msg) return {"error": msg, "scan_id": self.scan.id} - def stop_scan(self): + async def _start_scan_task(self, scan): + self.scan = scan + try: + await scan.start_without_generator() + except bbot.core.errors.ScanError as e: + log.error(f"Scan error: {e}") + log.trace(traceback.format_exc()) + except Exception: + log.critical(f"Encountered error: {traceback.format_exc()}") + self.on_scan_status("FAILED", scan.id) + finally: + self.task = None + + async def stop_scan(self): log.warning("Stopping scan") try: - with self._scan_lock: + async with self._scan_lock: if self.scan is None: msg = "Scan not in progress" log.warning(msg) @@ -147,40 +197,28 @@ def stop_scan(self): log.trace(traceback.format_exc()) finally: self.scan = None - self.thread = None + self.task = None - def scan_status(self): - with self._scan_lock: + async def scan_status(self): + async with self._scan_lock: if self.scan is None: - self.thread = None msg = "Scan not in progress" log.warning(msg) return {"error": msg} return {"success": "Polled scan", "scan_status": self.scan.status} - def on_scan_status(self, status, scan_id): - self.send({"message_type": "scan_status_change", "status": str(status), "scan_id": scan_id}) + async def on_scan_status(self, status, scan_id): + await self.send({"message_type": "scan_status_change", "status": str(status), "scan_id": scan_id}) - def on_scan_finish(self, scan): + async def on_scan_finish(self, scan): self.scan = None - self.thread = None + self.task = None - @staticmethod - def err_handle(callback, *args, **kwargs): + async def err_handle(self, callback, *args, **kwargs): try: - return callback(*args, **kwargs) + return await callback(*args, **kwargs) except Exception as e: msg = f"Error in {callback.__qualname__}(): {e}" log.error(msg) log.trace(traceback.format_exc()) return {"error": msg} - - def _start_scan(self, scan): - try: - scan.start_without_generator() - except bbot.core.errors.ScanError as e: - log.error(f"Scan error: {e}") - log.trace(traceback.format_exc()) - except Exception: - log.critical(f"Encountered error: {traceback.format_exc()}") - self.on_scan_status("FAILED", scan.id) diff --git a/bbot/agent/messages.py b/bbot/agent/messages.py index 3530fed114..34fd2c15c9 100644 --- a/bbot/agent/messages.py +++ b/bbot/agent/messages.py @@ -1,11 +1,12 @@ from uuid import UUID +from typing import Optional from pydantic import BaseModel class Message(BaseModel): conversation: UUID command: str - arguments: dict + arguments: Optional[dict] = {} ### COMMANDS ### @@ -17,6 +18,7 @@ class start_scan(BaseModel): modules: list output_modules: list = [] config: dict = {} + name: Optional[str] = None class stop_scan(BaseModel): diff --git a/bbot/cli.py b/bbot/cli.py index 8538e269d2..bc292f7c75 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -20,7 +20,7 @@ from bbot.modules import module_loader from bbot.core.configurator.args import parser from bbot.core.helpers.logger import log_to_stderr -from bbot.core.configurator import ensure_config_files, check_cli_args +from bbot.core.configurator import ensure_config_files, check_cli_args, environ log = logging.getLogger("bbot.cli") sys.stdout.reconfigure(line_buffering=True) @@ -39,6 +39,7 @@ async def _main(): global err global scan_name + environ.cli_execution = True ensure_config_files() @@ -70,7 +71,7 @@ async def _main(): agent = Agent(config) success = agent.setup() if success: - agent.start() + await agent.start() else: from bbot.scanner import Scanner @@ -293,9 +294,6 @@ def keyboard_listen(): log_to_stderr(str(e), level="ERROR") except Exception: raise - finally: - with suppress(NameError): - await scanner.cleanup() except bbot.core.errors.BBOTError as e: log_to_stderr(f"{e} (--debug for details)", level="ERROR") diff --git a/bbot/core/configurator/environ.py b/bbot/core/configurator/environ.py index c405037b14..23c7e78627 100644 --- a/bbot/core/configurator/environ.py +++ b/bbot/core/configurator/environ.py @@ -8,6 +8,10 @@ from ..helpers.misc import cpu_architecture, os_platform, os_platform_friendly +# keep track of whether BBOT is being executed via the CLI +cli_execution = False + + def flatten_config(config, base="bbot"): """ Flatten a JSON-like config into a list of environment variables: @@ -76,7 +80,7 @@ def prepare_environment(bbot_config): os.environ["BBOT_CPU_ARCH"] = cpu_architecture() # exchange certain options between CLI args and config - if args.cli_options is not None: + if cli_execution and args.cli_options is not None: # deps bbot_config["retry_deps"] = args.cli_options.retry_deps bbot_config["force_deps"] = args.cli_options.force_deps @@ -88,6 +92,17 @@ def prepare_environment(bbot_config): if args.cli_options.output_dir: bbot_config["output_dir"] = args.cli_options.output_dir + import logging + + log = logging.getLogger() + if bbot_config.get("debug", False): + global _log_level_override + bbot_config["silent"] = False + _log_level_override = logging.DEBUG + log = logging.getLogger("bbot") + log.setLevel(logging.DEBUG) + logging.getLogger("asyncio").setLevel(logging.DEBUG) + # copy config to environment bbot_environ = flatten_config(bbot_config) os.environ.update(bbot_environ) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index fda9cc8f2a..4f5bc3d615 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -111,9 +111,9 @@ def split_host_port(d): host = None with suppress(ValueError): if parsed.port is None: - if parsed.scheme == "https": + if parsed.scheme in ("https", "wss"): port = 443 - elif parsed.scheme == "http": + elif parsed.scheme in ("http", "ws"): port = 80 else: port = int(parsed.port) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 4d1f626541..2e06ae3381 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -297,6 +297,7 @@ async def _setup(self): self.debug(f"Finished setting up module {self.name}") except Exception as e: self.set_error_state() + # soft-fail if it's only a wordlist error if isinstance(e, WordlistError): status = None msg = f"{e}" diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index 3cfe82ea77..207bbe99cb 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -13,17 +13,17 @@ class ipneighbor(BaseModule): scope_distance_modifier = 1 _scope_shepherding = False - def setup(self): + async def setup(self): self.processed = set() self.num_bits = max(1, int(self.config.get("num_bits", 4))) return True - def filter_event(self, event): + async def filter_event(self, event): if str(event.module) in ("speculate", "ipneighbor"): return False return True - def handle_event(self, event): + async def handle_event(self, event): main_ip = event.host netmask = main_ip.max_prefixlen - min(main_ip.max_prefixlen, self.num_bits) network = ipaddress.ip_network(f"{main_ip}/{netmask}", strict=False) diff --git a/bbot/scanner/dispatcher.py b/bbot/scanner/dispatcher.py index 763da4c62e..a9c56c2b72 100644 --- a/bbot/scanner/dispatcher.py +++ b/bbot/scanner/dispatcher.py @@ -1,3 +1,9 @@ +import logging +import traceback + +log = logging.getLogger("bbot.scanner.dispatcher") + + class Dispatcher: """ Enables custom hooks/callbacks on certain scan events @@ -6,14 +12,21 @@ class Dispatcher: def set_scan(self, scan): self.scan = scan - def on_start(self, scan): + async def on_start(self, scan): return - def on_finish(self, scan): + async def on_finish(self, scan): return - def on_status(self, status, scan_id): + async def on_status(self, status, scan_id): """ Execute an event when the scan's status is updated """ self.scan.debug(f"Setting scan status to {status}") + + async def catch(self, callback, *args, **kwargs): + try: + return await callback(*args, **kwargs) + except Exception as e: + log.error(f"Error in {callback.__qualname__}(): {e}") + log.trace(traceback.format_exc()) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index d66330819d..c2459ad1cd 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -21,10 +21,10 @@ from bbot.core.event import make_event from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.helper import ConfigAwareHelper -from bbot.core.logger import init_logging, get_log_level from bbot.core.helpers.names_generator import random_name from bbot.core.configurator.environ import prepare_environment from bbot.core.errors import BBOTError, ScanError, ValidationError +from bbot.core.logger import init_logging, get_log_level, set_log_level log = logging.getLogger("bbot.scanner") @@ -74,6 +74,8 @@ def __init__( config = OmegaConf.create(config) self.config = OmegaConf.merge(bbot_config, config) prepare_environment(self.config) + if self.config.get("debug", False): + set_log_level(logging.DEBUG) self.strict_scope = strict_scope self.force_start = force_start @@ -198,18 +200,17 @@ async def start_without_generator(self): pass async def start(self): - await self.prep() - failed = True + scan_start_time = datetime.now() + try: + await self.prep() - if not self.target: - self.warning(f"No scan targets specified") + if not self.target: + self.warning(f"No scan targets specified") - # start status ticker - self.ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) + # start status ticker + self.ticker_task = asyncio.create_task(self._status_ticker(self.status_frequency)) - scan_start_time = datetime.now() - try: self.status = "STARTING" if not self.modules: @@ -219,7 +220,7 @@ async def start(self): else: self.hugesuccess(f"Starting scan {self.name}") - self.dispatcher.on_start(self) + await self.dispatcher.on_start(self) # start manager worker loops self.manager_worker_loop_tasks = [ @@ -292,7 +293,7 @@ async def start(self): scan_run_time = self.helpers.human_timedelta(scan_run_time) log_fn(f"Scan {self.name} completed in {scan_run_time} with status {self.status}") - self.dispatcher.on_finish(self) + await self.dispatcher.on_finish(self) def start_modules(self): self.verbose(f"Starting module worker loops") @@ -455,9 +456,12 @@ def status(self, status): if self.status == "ABORTING" and not status == "ABORTED": self.debug(f'Attempt to set invalid status "{status}" on aborted scan') else: - self._status = status - self._status_code = self._status_codes[status] - self.dispatcher.on_status(self._status, self.id) + if status != self._status: + self._status = status + self._status_code = self._status_codes[status] + asyncio.create_task(self.dispatcher.catch(self.dispatcher.on_status, self._status, self.id)) + else: + self.debug(f'Scan status is already "{status}"') else: self.debug(f'Attempt to set invalid status "{status}" on scan') diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 6ad61b914b..0306b3819b 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -298,36 +298,14 @@ class bbot_events: @pytest.fixture -def agent(monkeypatch, websocketapp, bbot_config): +def agent(monkeypatch, bbot_config): from bbot import agent - from bbot.modules.output.websocket import Websocket - - monkeypatch.setattr(Websocket, "send", lambda *args, **kwargs: True) test_agent = agent.Agent(bbot_config) test_agent.setup() - monkeypatch.setattr(test_agent, "ws", websocketapp()) return test_agent -@pytest.fixture -def websocketapp(): - class WebSocketApp: - def __init__(*args, **kwargs): - return - - def send(self, message): - assert type(message) == str - - def run_forever(*args, **kwargs): - return False - - def close(self): - return - - return WebSocketApp - - # bbot config from bbot import config as default_config diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 5e534f5067..80daae5a53 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -33,7 +33,7 @@ dns_debug: true user_agent: "BBOT Test User-Agent" http_debug: false keep_scans: 1 -agent_url: test +agent_url: ws://127.0.0.1:8765 agent_token: test dns_resolution: false speculate: false diff --git a/bbot/test/test_step_2/test_agent.py b/bbot/test/test_step_2/test_agent.py index e64d3ed162..d5fc2b35ba 100644 --- a/bbot/test/test_step_2/test_agent.py +++ b/bbot/test/test_step_2/test_agent.py @@ -1,17 +1,140 @@ -from time import sleep +import json +import asyncio from ..bbot_fixtures import * # noqa: F401 -def test_agent(agent): - agent.start() - agent.on_error(agent.ws, "test") - agent.on_close(agent.ws, "test", "test") - agent.on_open(agent.ws) - agent.on_message( - agent.ws, - '{"conversation": "90196cc1-299f-4555-82a0-bc22a4247590", "command": "start_scan", "arguments": {"scan_id": "90196cc1-299f-4555-82a0-bc22a4247590", "targets": ["www.blacklanternsecurity.com"], "modules": ["ipneighbor"], "output_modules": ["human"]}}', - ) - sleep(0.5) - agent.scan_status() - agent.stop_scan() +_first_run = True +success = False +scan_done = asyncio.Event() + + +async def websocket_handler(websocket, path): + # whether this is the first run + global _first_run + first_run = int(_first_run) + # whether the test succeeded + global success + # test phase + phase = "ping" + # control channel or event channel? + control = True + + if path == "/" and first_run: + # test ping + await websocket.send(json.dumps({"conversation": "90196cc1-299f-4555-82a0-bc22a4247590", "command": "ping"})) + _first_run = False + else: + control = False + + # Bearer token + assert websocket.request_headers["Authorization"] == "Bearer test" + + async for message in websocket: + log.debug(f"PHASE: {phase}, MESSAGE: {message}") + if not control or not first_run: + continue + m = json.loads(message) + # ping + if phase == "ping": + assert json.loads(message)["message_type"] == "pong" + phase = "start_scan_bad" + if phase == "start_scan_bad": + await websocket.send( + json.dumps( + { + "conversation": "90196cc1-299f-4555-82a0-bc22a4247590", + "command": "start_scan", + "arguments": { + "scan_id": "90196cc1-299f-4555-82a0-bc22a4247590", + "targets": ["127.0.0.2"], + "modules": ["asdf"], + "output_modules": ["human"], + "name": "agent_test_scan_bad", + }, + } + ) + ) + phase = "success" + continue + # scan start success + if phase == "success": + assert m["message"]["success"] == "Started scan" + phase = "cleaning_up" + continue + # CLEANING_UP status message + if phase == "cleaning_up": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "CLEANING_UP" + phase = "failed" + continue + # FAILED status message + if phase == "failed": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "FAILED" + phase = "start_scan" + # start good scan + if phase == "start_scan": + await websocket.send( + json.dumps( + { + "conversation": "90196cc1-299f-4555-82a0-bc22a4247590", + "command": "start_scan", + "arguments": { + "scan_id": "90196cc1-299f-4555-82a0-bc22a4247590", + "targets": ["127.0.0.2"], + "modules": ["ipneighbor"], + "output_modules": ["human"], + "name": "agent_test_scan", + }, + } + ) + ) + phase = "success_2" + continue + # scan start success + if phase == "success_2": + assert m["message"]["success"] == "Started scan" + phase = "starting" + continue + # STARTING status message + if phase == "starting": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "STARTING" + phase = "running" + continue + # RUNNING status message + if phase == "running": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "RUNNING" + phase = "finishing" + continue + # FINISHING status message + if phase == "finishing": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "FINISHING" + phase = "cleaning_up_2" + continue + # CLEANING_UP status message + if phase == "cleaning_up_2": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "CLEANING_UP" + phase = "finished_2" + continue + # FINISHED status message + if phase == "finished_2": + assert m["message_type"] == "scan_status_change" + assert m["status"] == "FINISHED" + success = True + scan_done.set() + break + + +@pytest.mark.asyncio +async def test_agent(agent): + global success + async with websockets.serve(websocket_handler, "127.0.0.1", 8765): + asyncio.create_task(agent.start()) + # wait for 30 seconds + await asyncio.wait_for(scan_done.wait(), 10) + assert success diff --git a/bbot/test/test_step_2/test_cloud_helpers.py b/bbot/test/test_step_2/test_cloud_helpers.py index 49f5a74109..c4494037bc 100644 --- a/bbot/test/test_step_2/test_cloud_helpers.py +++ b/bbot/test/test_step_2/test_cloud_helpers.py @@ -1,9 +1,10 @@ from ..bbot_fixtures import * # noqa: F401 -def test_cloud_helpers(monkeypatch, bbot_scanner, bbot_config): +@pytest.mark.asyncio +async def test_cloud_helpers(monkeypatch, bbot_scanner, bbot_config): scan1 = bbot_scanner("127.0.0.1", config=bbot_config) - scan1.load_modules() + await scan1.load_modules() aws_event1 = scan1.make_event("amazonaws.com", source=scan1.root_event) aws_event2 = scan1.make_event("asdf.amazonaws.com", source=scan1.root_event) aws_event3 = scan1.make_event("asdfamazonaws.com", source=scan1.root_event) diff --git a/bbot/test/test_step_2/test_config.py b/bbot/test/test_step_2/test_config.py index 3fa7961f27..2d9980a2c7 100644 --- a/bbot/test/test_step_2/test_config.py +++ b/bbot/test/test_step_2/test_config.py @@ -1,9 +1,10 @@ from ..bbot_fixtures import * # noqa: F401 -def test_config(bbot_config, bbot_scanner): +@pytest.mark.asyncio +async def test_config(bbot_config, bbot_scanner): scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor", "speculate"], config=bbot_config) - scan1.load_modules() + await scan1.load_modules() assert scan1.config.plumbus == "asdf" assert scan1.modules["ipneighbor"].config.test_option == "ipneighbor" assert scan1.modules["python"].config.test_option == "asdf" From b8ee5ad561587434e63da422c704a11db448b1cb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 May 2023 14:20:47 -0400 Subject: [PATCH 021/387] fix NameError --- bbot/test/test_step_2/test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/test_step_2/test_agent.py b/bbot/test/test_step_2/test_agent.py index d5fc2b35ba..63121d6d19 100644 --- a/bbot/test/test_step_2/test_agent.py +++ b/bbot/test/test_step_2/test_agent.py @@ -1,5 +1,6 @@ import json import asyncio +import websockets from ..bbot_fixtures import * # noqa: F401 From 8683f042f0c5c2cf583494e5976e880a7fa8562f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 May 2023 15:12:23 -0400 Subject: [PATCH 022/387] manager tests passing --- bbot/scanner/manager.py | 1 + bbot/test/bbot_fixtures.py | 12 +++- bbot/test/test_step_2/test_agent.py | 1 - bbot/test/test_step_2/test_events.py | 3 +- bbot/test/test_step_2/test_manager.py | 74 +++++++++++++++--------- bbot/test/test_step_2/test_python_api.py | 1 + bbot/test/test_step_2/test_scan.py | 3 +- bbot/test/test_step_2/test_scope.py | 10 ++-- 8 files changed, 67 insertions(+), 38 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 7538bb89c2..d61fa5b14b 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -237,6 +237,7 @@ async def _emit_event(self, event, *args, **kwargs): if self.dns_resolution: emit_children = -1 < event.scope_distance < self.scan.dns_search_distance if emit_children: + # only emit DNS children once for each unique host host_hash = hash(str(event.host)) if host_hash in self.events_accepted: emit_children = False diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 0306b3819b..5b8d2ffe0e 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -1,6 +1,7 @@ import os import sys import pytest +import asyncio # noqa import logging import subprocess import tldextract @@ -71,6 +72,8 @@ def patch_commands(): # ffuf """{"input":{"FUZZ":"L2luZGV4Lmh0bWw="},"position":1,"status":200,"length":1256,"words":298,"lines":47,"content-type":"text/html;charset=UTF-8","redirectlocation":"","url":"http://example.com:80//index.html","duration":101243249,"resultfile":"","host":"example.com:80"}""", "https://api.publicapis.org:443/health", + # fingerprintx + """{"ip":"8.8.8.8","port":443,"protocol":"https","tls":true,"transport":"tcp","version":"HTTP server (unknown)","metadata":{"status":"302 Found","statusCode":302,"responseHeaders":{"Access-Control-Allow-Origin":["*"],"Alt-Svc":["h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000"],"Content-Length":["216"],"Content-Type":["text/html; charset=UTF-8"],"Date":["Mon, 15 May 2023 18:34:49 GMT"],"Location":["https://dns.google/"],"Server":["HTTP server (unknown)"],"X-Content-Type-Options":["nosniff"],"X-Frame-Options":["SAMEORIGIN"],"X-Xss-Protection":["0"]},"technologies":["HTTP/3"]}}""" # open port "api.publicapis.org:443", # host @@ -80,10 +83,11 @@ def patch_commands(): ] def patch_scan_commands(scanner): - def run(*args, **kwargs): + async def run(*args, **kwargs): log.debug(f"helpers.command.run(args={args}, kwargs={kwargs})") text = kwargs.get("text", True) - return subprocess.run(["echo", "\n".join(sample_output)], text=text, stdout=subprocess.PIPE) + output = "\n".join(sample_output) + return subprocess.run(["echo", output], text=text) def run_live(*args, **kwargs): log.debug(f"helpers.command.run_live(args={args}, kwargs={kwargs})") @@ -156,7 +160,6 @@ def scan(monkeypatch, patch_ansible, patch_commands, bbot_config): bbot_scan = Scanner("127.0.0.1", modules=["ipneighbor"], config=bbot_config) patch_commands(bbot_scan) patch_ansible(bbot_scan) - bbot_scan.status = "RUNNING" fallback_nameservers_file = bbot_scan.helpers.bbot_home / "fallback_nameservers.txt" with open(fallback_nameservers_file, "w") as f: @@ -312,6 +315,9 @@ def agent(monkeypatch, bbot_config): test_config = OmegaConf.load(Path(__file__).parent / "test.conf") test_config = OmegaConf.merge(default_config, test_config) +if test_config.get("debug", False): + logging.getLogger("bbot").setLevel(logging.DEBUG) + @pytest.fixture def bbot_config(): diff --git a/bbot/test/test_step_2/test_agent.py b/bbot/test/test_step_2/test_agent.py index 63121d6d19..a673effeae 100644 --- a/bbot/test/test_step_2/test_agent.py +++ b/bbot/test/test_step_2/test_agent.py @@ -1,5 +1,4 @@ import json -import asyncio import websockets from ..bbot_fixtures import * # noqa: F401 diff --git a/bbot/test/test_step_2/test_events.py b/bbot/test/test_step_2/test_events.py index f184f91b60..03ebde6354 100644 --- a/bbot/test/test_step_2/test_events.py +++ b/bbot/test/test_step_2/test_events.py @@ -5,7 +5,8 @@ from ..bbot_fixtures import * -def test_events(events, scan, helpers, bbot_config): +@pytest.mark.asyncio +async def test_events(events, scan, helpers, bbot_config): assert events.ipv4.type == "IP_ADDRESS" assert events.ipv6.type == "IP_ADDRESS" assert events.netv4.type == "IP_RANGE" diff --git a/bbot/test/test_step_2/test_manager.py b/bbot/test/test_step_2/test_manager.py index eff5190e4d..6b43ce07d8 100644 --- a/bbot/test/test_step_2/test_manager.py +++ b/bbot/test/test_step_2/test_manager.py @@ -1,7 +1,8 @@ from ..bbot_fixtures import * # noqa: F401 -def test_manager(bbot_config, bbot_scanner): +@pytest.mark.asyncio +async def test_manager(bbot_config, bbot_scanner): dns_config = OmegaConf.merge( default_config, OmegaConf.create({"dns_resolution": True, "scope_report_distance": 1}) ) @@ -9,14 +10,25 @@ def test_manager(bbot_config, bbot_scanner): # test _emit_event results = [] output = [] + event_children = [] + + async def results_append(e): + results.append(e) + + async def output_append(e): + output.append(e) + + def event_children_append(e): + event_children.append(e) + success_callback = lambda e: results.append("success") scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor"], output_modules=["human"], config=dns_config) - scan1.load_modules() + await scan1.load_modules() module = scan1.modules["ipneighbor"] module.scope_distance_modifier = 0 - module.queue_event = lambda e: results.append(e) + module.queue_event = results_append output_module = scan1.modules["human"] - output_module.queue_event = lambda e: output.append(e) + output_module.queue_event = output_append scan1.status = "RUNNING" manager = scan1.manager # manager.distribute_event = lambda e: results.append(e) @@ -36,11 +48,17 @@ class DummyModule3: localhost.module = DummyModule1() # make sure abort_if works as intended - manager._emit_event(localhost, abort_if=lambda e: e.module._type == "output") + await manager._emit_event(localhost, abort_if=lambda e: e.module._type == "output") assert len(results) == 0 manager.events_accepted.clear() + manager.events_distributed.clear() + await manager._emit_event(localhost, abort_if=lambda e: e.module._type != "output") + assert len(results) == 1 + results.clear() + manager.events_accepted.clear() + manager.events_distributed.clear() # make sure success_callback works as intended - manager._emit_event( + await manager._emit_event( localhost, on_success_callback=success_callback, abort_if=lambda e: e.module._type == "plumbus" ) assert localhost in results @@ -55,9 +73,8 @@ class DummyModule3: googledns.module = DummyModule2() googledns.source = "asdf" googledns.make_in_scope() - event_children = [] - manager.emit_event = lambda e, *args, **kwargs: event_children.append(e) - manager._emit_event(googledns) + manager.queue_event = event_children_append + await manager._emit_event(googledns) assert len(event_children) > 0 assert googledns in results assert googledns in output @@ -65,7 +82,7 @@ class DummyModule3: output.clear() event_children.clear() # make sure deduplication catches the same event - manager._emit_event(googledns) + await manager._emit_event(googledns) assert len(output) == 0 assert len(results) == 0 assert len(event_children) == 0 @@ -73,7 +90,7 @@ class DummyModule3: event_children.clear() # make sure _force_output overrides dup detection googledns._force_output = True - manager._emit_event(googledns) + await manager._emit_event(googledns) assert googledns in output assert len(event_children) == 0 googledns._force_output = False @@ -83,7 +100,7 @@ class DummyModule3: source_event = manager.scan.make_event("1.2.3.4", "IP_ADDRESS", source=manager.scan.root_event) source_event._resolved.set() googledns.source = source_event - manager._emit_event(googledns) + await manager._emit_event(googledns) assert len(event_children) == 0 assert googledns in output @@ -91,21 +108,22 @@ class DummyModule3: msg = "Ignore this error, it belongs here" exceptions = (Exception(msg), KeyboardInterrupt(msg), BrokenPipeError(msg)) for e in exceptions: - with manager.catch(): + with manager.scan.catch(): raise e -def test_scope_distance(bbot_scanner, bbot_config): +@pytest.mark.asyncio +async def test_scope_distance(bbot_scanner, bbot_config): # event filtering based on scope_distance scan1 = bbot_scanner( "127.0.0.1", "evilcorp.com", modules=["ipneighbor"], output_modules=["json"], config=bbot_config ) scan1.status = "RUNNING" - scan1.load_modules() + await scan1.load_modules() module = scan1.modules["ipneighbor"] - module_queue = module.incoming_event_queue.queue + module_queue = module.incoming_event_queue._queue output_module = scan1.modules["json"] - output_queue = output_module.incoming_event_queue.queue + output_queue = output_module.incoming_event_queue._queue manager = scan1.manager test_event1 = scan1.make_event("127.0.0.1", source=scan1.root_event) @@ -114,7 +132,7 @@ def test_scope_distance(bbot_scanner, bbot_config): assert module.scope_distance_modifier == 1 # test _emit_event() with scope_distance == 0 - manager._emit_event(test_event1) + await manager._emit_event(test_event1) assert test_event1.scope_distance == 0 assert test_event1._internal == False assert test_event1 in output_queue @@ -127,41 +145,41 @@ def test_scope_distance(bbot_scanner, bbot_config): dns_event = scan1.make_event("evilcorp.com", source=scan1.root_event) # non-watched event type - manager._emit_event(dns_event) + await manager._emit_event(dns_event) assert dns_event.scope_distance == 0 assert dns_event in output_queue assert dns_event not in module_queue # test _emit_event() with scope_distance == 1 assert test_event2.scope_distance == 1 - manager._emit_event(test_event2) + await manager._emit_event(test_event2) assert test_event2.scope_distance == 1 assert test_event2._internal == True assert test_event2 not in output_queue assert test_event2 in module_queue - valid, reason = module._event_postcheck(test_event2) + valid, reason = await module._event_postcheck(test_event2) assert valid # test _emit_event() with scope_distance == 2 assert test_event3.scope_distance == 2 - manager._emit_event(test_event3) + await manager._emit_event(test_event3) assert test_event3.scope_distance == 2 assert test_event3._internal == True assert test_event3 not in output_queue assert test_event3 in module_queue - valid, reason = module._event_postcheck(test_event3) + valid, reason = await module._event_postcheck(test_event3) assert not valid assert reason.startswith("its scope_distance (2) exceeds the maximum allowed by the scan") # test _emit_event() with scope_distance == 2 and _force_output == True assert test_event4.scope_distance == 2 - manager._emit_event(test_event4) + await manager._emit_event(test_event4) assert test_event4.scope_distance == 2 assert test_event4._internal == True assert test_event4._force_output == True assert test_event4 in output_queue assert test_event4 in module_queue - valid, reason = module._event_postcheck(test_event4) + valid, reason = await module._event_postcheck(test_event4) assert not valid assert reason.startswith("its scope_distance (2) exceeds the maximum allowed by the scan") @@ -170,7 +188,7 @@ def test_scope_distance(bbot_scanner, bbot_config): assert geoevent.scope_distance == 3 assert geoevent.always_emit == True assert geoevent._force_output == False - manager._emit_event(geoevent) + await manager._emit_event(geoevent) assert geoevent._force_output == True assert geoevent in output_queue assert geoevent not in module_queue @@ -183,10 +201,10 @@ def test_scope_distance(bbot_scanner, bbot_config): assert affiliate_event._always_emit == False assert affiliate_event.always_emit == True assert affiliate_event._force_output == False - manager._emit_event(affiliate_event) + await manager._emit_event(affiliate_event) assert affiliate_event._force_output == True assert affiliate_event in output_queue assert affiliate_event in module_queue - valid, reason = module._event_postcheck(affiliate_event) + valid, reason = await module._event_postcheck(affiliate_event) assert not valid assert reason.startswith("its scope_distance (3) exceeds the maximum allowed by the scan") diff --git a/bbot/test/test_step_2/test_python_api.py b/bbot/test/test_step_2/test_python_api.py index 00b26f3a6e..f4ef428af9 100644 --- a/bbot/test/test_step_2/test_python_api.py +++ b/bbot/test/test_step_2/test_python_api.py @@ -1,6 +1,7 @@ from ..bbot_fixtures import * +@pytest.mark.asyncio def test_python_api(bbot_config): from bbot.scanner import Scanner diff --git a/bbot/test/test_step_2/test_scan.py b/bbot/test/test_step_2/test_scan.py index 6e60c2190e..2bde71a382 100644 --- a/bbot/test/test_step_2/test_scan.py +++ b/bbot/test/test_step_2/test_scan.py @@ -1,7 +1,8 @@ from ..bbot_fixtures import * -def test_scan( +@pytest.mark.asyncio +async def test_scan( patch_ansible, patch_commands, events, diff --git a/bbot/test/test_step_2/test_scope.py b/bbot/test/test_step_2/test_scope.py index 2fe92373b2..55838d3458 100644 --- a/bbot/test/test_step_2/test_scope.py +++ b/bbot/test/test_step_2/test_scope.py @@ -36,11 +36,13 @@ def check_events(self, events): return True -def test_scope_blacklist(bbot_config, bbot_scanner, bbot_httpserver): +@pytest.mark.asyncio +async def test_scope_blacklist(bbot_config, bbot_scanner, bbot_httpserver): x = Scope_test_blacklist(bbot_config, bbot_scanner, bbot_httpserver, module_name="httpx") - x.run() + await x.run() -def test_scope_whitelist(bbot_config, bbot_scanner, bbot_httpserver): +@pytest.mark.asyncio +async def test_scope_whitelist(bbot_config, bbot_scanner, bbot_httpserver): x = Scope_test_whitelist(bbot_config, bbot_scanner, bbot_httpserver, module_name="httpx") - x.run() + await x.run() From 47a659aaaaedc5c6396b0e448677d4b304d042ce Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 15 May 2023 16:52:02 -0400 Subject: [PATCH 023/387] working through asyncifying modules --- bbot/core/helpers/depsinstaller/installer.py | 6 +- bbot/modules/azure_tenant.py | 10 +-- bbot/modules/base.py | 2 +- bbot/modules/bevigil.py | 18 +++--- bbot/modules/binaryedge.py | 10 +-- bbot/modules/builtwith.py | 16 ++--- bbot/modules/c99.py | 8 +-- bbot/modules/censys.py | 20 +++--- bbot/modules/crt.py | 6 +- bbot/modules/dnscommonsrv.py | 8 +-- bbot/modules/dnszonetransfer.py | 14 ++--- bbot/modules/fullhunt.py | 10 +-- bbot/modules/generic_ssrf.py | 43 ++++++------- bbot/modules/github.py | 66 +++++++++++--------- bbot/modules/host_header.py | 23 +++---- bbot/modules/httpx.py | 2 +- bbot/modules/ipstack.py | 8 +-- bbot/modules/ntlm.py | 49 +++++++-------- bbot/modules/output/asset_inventory.py | 4 +- bbot/modules/output/http.py | 56 +++++++++-------- bbot/modules/report/affiliates.py | 6 +- bbot/modules/shodan_dns.py | 10 +-- bbot/modules/sslcert.py | 2 +- bbot/modules/viewdns.py | 12 ++-- bbot/test/test_step_2/test_modules_basic.py | 13 +++- 25 files changed, 218 insertions(+), 204 deletions(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 2195237f38..72f6b5575c 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -165,9 +165,9 @@ async def pip_install(self, packages, constraints=None): try: process = await self.parent_helper.run(command, check=True) message = f'Successfully installed pip packages "{packages_str}"' - output = process.stdout.splitlines()[-1] - if output: - message = output + output = process.stdout + if output is not None: + message = output.splitlines()[-1] log.info(message) return True except CalledProcessError as err: diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py index fafa391a76..b2626810e1 100644 --- a/bbot/modules/azure_tenant.py +++ b/bbot/modules/azure_tenant.py @@ -12,14 +12,14 @@ class azure_tenant(viewdns): base_url = "https://autodiscover-s.outlook.com" in_scope_only = True - def setup(self): + async def setup(self): self.processed = set() self.d_xml_regex = re.compile(r"([^<>/]*)", re.I) return True - def handle_event(self, event): + async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) - domains, _ = self.query(query) + domains, _ = await self.query(query) if domains: self.success(f'Found {len(domains):,} domains under tenant for "{query}"') for domain in domains: @@ -27,7 +27,7 @@ def handle_event(self, event): self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) # todo: tenants? - def query(self, domain): + async def query(self, domain): url = f"{self.base_url}/autodiscover/autodiscover.svc" data = f""" @@ -56,7 +56,7 @@ def query(self, domain): self.debug(f"Retrieving tenant domains at {url}") - r = self.request_with_fail_count(url, method="POST", headers=headers, data=data) + r = await self.request_with_fail_count(url, method="POST", headers=headers, data=data) status_code = getattr(r, "status_code", 0) if status_code not in (200, 421): self.warning(f'Error retrieving azure_tenant domains for "{domain}" (status code: {status_code})') diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 2e06ae3381..e359ed7816 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -173,7 +173,7 @@ def require_api_key(self): else: return None, "No API key set" - def ping(self): + async def ping(self): """ Used in conjuction with require_api_key to ensure an API is up and responding diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index d81a081a91..a24d6331c1 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -15,35 +15,35 @@ class bevigil(shodan_dns): base_url = "https://osint.bevigil.com/api" - def setup(self): + async def setup(self): self.api_key = self.config.get("api_key", "") self.headers = {"X-Access-Token": self.api_key} self.urls = self.config.get("urls", False) return super().setup() - def ping(self): + async def ping(self): pass - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - subdomains = self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains) + subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains) if subdomains: for subdomain in subdomains: self.emit_event(subdomain, "DNS_NAME", source=event) if self.urls: - urls = self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls) + urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls) if urls: for parsed_url in self.helpers.collapse_urls(urls): self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", source=event) - def request_subdomains(self, query): + async def request_subdomains(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/" - return self.request_with_fail_count(url, headers=self.headers) + return await self.request_with_fail_count(url, headers=self.headers) - def request_urls(self, query): + async def request_urls(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}/urls/" - return self.request_with_fail_count(url, headers=self.headers) + return await self.request_with_fail_count(url, headers=self.headers) def parse_subdomains(self, r, query=None): results = set() diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py index 7bcb266806..497dd03069 100644 --- a/bbot/modules/binaryedge.py +++ b/bbot/modules/binaryedge.py @@ -14,20 +14,20 @@ class binaryedge(shodan_dns): base_url = "https://api.binaryedge.io/v2" - def setup(self): + async def setup(self): self.max_records = self.config.get("max_records", 1000) self.headers = {"X-Key": self.config.get("api_key", "")} return super().setup() - def ping(self): + async def ping(self): url = f"{self.base_url}/user/subscription" - j = self.request_with_fail_count(url, headers=self.headers).json() + j = (await self.request_with_fail_count(url, headers=self.headers)).json() assert j.get("requests_left", 0) > 0 - def request_url(self, query): + async def request_url(self, query): # todo: host query (certs + services) url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}" - return self.request_with_fail_count(url, headers=self.headers) + return await self.request_with_fail_count(url, headers=self.headers) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 8a920f6a24..e8139d5dd8 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -22,33 +22,33 @@ class builtwith(shodan_dns): options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"} base_url = "https://api.builtwith.com" - def ping(self): + async def ping(self): # builtwith does not have a ping feature, so we skip it to save API credits return - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) # domains - subdomains = self.query(query, parse_fn=self.parse_domains, request_fn=self.request_domains) + subdomains = await self.query(query, parse_fn=self.parse_domains, request_fn=self.request_domains) if subdomains: for s in subdomains: if s != event: self.emit_event(s, "DNS_NAME", source=event) # redirects if self.config.get("redirects", True): - redirects = self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects) + redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects) if redirects: for r in redirects: if r != event: self.emit_event(r, "DNS_NAME", source=event, tags=["affiliate"]) - def request_domains(self, query): + async def request_domains(self, query): url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) - def request_redirects(self, query): + async def request_redirects(self, query): url = f"{self.base_url}/redirect1/api.json?KEY={self.api_key}&LOOKUP={query}" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) def parse_domains(self, r, query): """ diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 7fde17dcd9..5b0179def4 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -11,14 +11,14 @@ class c99(shodan_dns): base_url = "https://api.c99.nl" - def ping(self): + async def ping(self): url = f"{self.base_url}/randomnumber?key={self.api_key}&between=1,100&json" - response = self.request_with_fail_count(url) + response = await self.request_with_fail_count(url) assert response.json()["success"] == True - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/subdomainfinder?key={self.api_key}&domain={self.helpers.quote(query)}&json" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py index ab6ca65a0b..ed713472d7 100644 --- a/bbot/modules/censys.py +++ b/bbot/modules/censys.py @@ -21,7 +21,7 @@ class censys(shodan_dns): deps_pip = ["censys~=2.1.9"] - def setup(self): + async def setup(self): self.max_records = self.config.get("max_records", 1000) self.api_id = self.config.get("api_id", "") self.api_secret = self.config.get("api_secret", "") @@ -32,13 +32,13 @@ def setup(self): self.certificates = CensysCertificates(api_id=self.api_id, api_secret=self.api_secret) return super().setup() - def ping(self): + async def ping(self): quota = self.certificates.quota() used = int(quota["used"]) allowance = int(quota["allowance"]) assert used < allowance, "No quota remaining" - def query(self, query): + async def query(self, query): emails = set() dns_names = set() ip_addresses = dict() @@ -46,9 +46,10 @@ def query(self, query): # certificates certificate_query = f"parsed.names: {query}" certificate_fields = ["parsed.names", "parsed.issuer_dn", "parsed.subject_dn"] - for result in self.certificates.search( - certificate_query, fields=certificate_fields, max_records=self.max_records - ): + results = await self.scan.run_in_executor( + self.certificates.search, certificate_query, fields=certificate_fields, max_records=self.max_records + ) + for result in results: parsed_names = result.get("parsed.names", []) # helps filter out third-party certs with a lot of garbage names _filter = lambda x: True @@ -64,7 +65,10 @@ def query(self, query): per_page = 100 pages = max(1, int(self.max_records / per_page)) hosts_query = f"services.tls.certificates.leaf_data.names: {query} or services.tls.certificates.leaf_data.subject.email_address: {query}" - for i, page in enumerate(self.hosts.search(hosts_query, per_page=per_page, pages=pages)): + hosts_results = await self.scan.run_in_executor( + self.hosts.search, hosts_query, per_page=per_page, pages=pages + ) + for i, page in enumerate(hosts_results): for result in page: ip = result.get("ip", "") if not ip: @@ -90,7 +94,7 @@ def query(self, query): return emails, dns_names, ip_addresses - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) emails, dns_names, ip_addresses = self.query(query) for email in emails: diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py index ca8aefed1f..a4410354ce 100644 --- a/bbot/modules/crt.py +++ b/bbot/modules/crt.py @@ -10,14 +10,14 @@ class crt(crobat): base_url = "https://crt.sh" reject_wildcards = False - def setup(self): + async def setup(self): self.cert_ids = set() return super().setup() - def request_url(self, query): + async def request_url(self, query): params = {"q": f"%.{query}", "output": "json"} url = self.helpers.add_get_params(self.base_url, params).geturl() - return self.request_with_fail_count(url, timeout=self.http_timeout + 10) + return await self.request_with_fail_count(url, timeout=self.http_timeout + 10) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 8333e8293b..ab8f9029e7 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -96,14 +96,14 @@ class dnscommonsrv(BaseModule): meta = {"description": "Check for common SRV records"} max_event_handlers = 10 - def filter_event(self, event): + async def filter_event(self, event): # skip SRV wildcards - if "SRV" in self.helpers.is_wildcard(event.host): + if "SRV" in await self.helpers.is_wildcard(event.host): return False return True - def handle_event(self, event): + async def handle_event(self, event): queries = [event.data] + [f"{srv}.{event.data}" for srv in common_srvs] - for query, results in self.helpers.resolve_batch(queries, type="srv"): + async for query, results in self.helpers.resolve_batch(queries, type="srv"): if results: self.emit_event(query, "DNS_NAME", tags=["srv-record"], source=event) diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index 6e56284f92..4a3fb5a58b 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -14,29 +14,29 @@ class dnszonetransfer(BaseModule): max_event_handlers = 5 suppress_dupes = False - def setup(self): + async def setup(self): self.timeout = self.config.get("timeout", 10) return True - def filter_event(self, event): - if any([x in event.tags for x in ("ns_record", "soa_record")]): + async def filter_event(self, event): + if any([x in event.tags for x in ("ns-record", "soa-record")]): return True return False - def handle_event(self, event): + async def handle_event(self, event): domain = event.data self.debug("Finding nameservers with NS/SOA query") - nameservers = list(self.helpers.resolve(event.data, type=("NS", "SOA"))) + nameservers = list(await self.helpers.resolve(event.data, type=("NS", "SOA"))) nameserver_ips = set() for n in nameservers: - nameserver_ips.update(self.helpers.resolve(n)) + nameserver_ips.update(await self.helpers.resolve(n)) self.debug(f"Found {len(nameservers):} nameservers for domain {domain}") for nameserver in nameserver_ips: if self.scan.stopping: break try: self.debug(f"Attempting zone transfer against {nameserver} for domain {domain}") - xfr_answer = dns.query.xfr(nameserver, domain, timeout=self.timeout, lifetime=self.timeout) + xfr_answer = await dns.asyncquery.xfr(nameserver, domain, timeout=self.timeout, lifetime=self.timeout) zone = dns.zone.from_xfr(xfr_answer) except Exception as e: self.debug(f"Error retrieving zone: {e}") diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index e0c051c561..4d6e740b09 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -11,20 +11,20 @@ class fullhunt(shodan_dns): base_url = "https://fullhunt.io/api/v1" - def setup(self): + async def setup(self): self.api_key = self.config.get("api_key", "") self.headers = {"x-api-key": self.api_key} return super().setup() - def ping(self): + async def ping(self): url = f"{self.base_url}/auth/status" - j = self.request_with_fail_count(url, headers=self.headers).json() + j = await self.request_with_fail_count(url, headers=self.headers).json() remaining = j["user_credits"]["remaining_credits"] assert remaining > 0, "No credits remaining" - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains" - return self.request_with_fail_count(url, headers=self.headers) + return await self.request_with_fail_count(url, headers=self.headers) def parse_results(self, r, query): return r.json().get("hosts", []) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index db0aa2a3c6..91e1721ab1 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -49,7 +49,7 @@ def set_base_url(self, event): def create_paths(self): return self.paths - def test(self, event): + async def test(self, event): base_url = self.set_base_url(event) for test_path in self.test_paths: @@ -59,7 +59,7 @@ def test(self, event): ) test_url = f"{base_url}{test_path_prepared}" self.parent_module.debug(f"Sending request to URL: {test_url}") - r = self.parent_module.helpers.curl(url=test_url) + r = await self.parent_module.helpers.curl(url=test_url) if r: self.process(event, r, subdomain_tag) @@ -104,7 +104,7 @@ class Generic_SSRF_POST(BaseSubmodule): def set_base_url(self, event): return event.data - def test(self, event): + async def test(self, event): test_url = f"{event.data}" subdomain_tag = self.parent_module.helpers.rand_string(4, digits=False) @@ -121,7 +121,7 @@ def test(self, event): post_data_list = [(subdomain_tag, post_data), (subdomain_tag_lower, post_data_lower)] for tag, pd in post_data_list: - r = self.parent_module.helpers.curl(url=test_url, method="POST", post_data=pd) + r = await self.parent_module.helpers.curl(url=test_url, method="POST", post_data=pd) self.process(event, r, tag) @@ -130,7 +130,7 @@ class Generic_XXE(BaseSubmodule): severity = "HIGH" paths = None - def test(self, event): + async def test(self, event): rand_entity = self.parent_module.helpers.rand_string(4, digits=False) subdomain_tag = self.parent_module.helpers.rand_string(4, digits=False) @@ -141,7 +141,7 @@ def test(self, event): ]> &{rand_entity};""" test_url = f"{event.parsed.scheme}://{event.parsed.netloc}/" - r = self.parent_module.helpers.curl( + r = await self.parent_module.helpers.curl( url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"} ) if r: @@ -157,7 +157,7 @@ class generic_ssrf(BaseModule): deps_apt = ["curl"] - def setup(self): + async def setup(self): self.submodules = {} self.interactsh_subdomain_tags = {} self.severity = None @@ -166,7 +166,7 @@ def setup(self): if self.scan.config.get("interactsh_disable", False) == False: try: self.interactsh_instance = self.helpers.interactsh() - self.interactsh_domain = self.interactsh_instance.register(callback=self.interactsh_callback) + self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback) except InteractshError as e: self.warning(f"Interactsh failure: {e}") return False @@ -184,12 +184,9 @@ def setup(self): return True - def handle_event(self, event): - self.test_submodules(self.submodules, event) - - def test_submodules(self, submodules, event, **kwargs): - for s in submodules.values(): - s.test(event, **kwargs) + async def handle_event(self, event): + for s in self.submodules.values(): + await s.test(event) def interactsh_callback(self, r): full_id = r.get("full-id", None) @@ -217,13 +214,11 @@ def interactsh_callback(self, r): # this is likely caused by something trying to resolve the base domain first and can be ignored self.debug("skipping result because subdomain tag was missing") - def finish(self): - from time import sleep - - sleep(5) - - try: - for r in self.interactsh_instance.poll(): - self.interactsh_callback(r) - except InteractshError as e: - self.debug(f"Error in interact.sh: {e}") + async def finish(self): + if self.scan.config.get("interactsh_disable", False) == False: + await self.helpers.sleep(5) + try: + for r in await self.interactsh_instance.poll(): + self.interactsh_callback(r) + except InteractshError as e: + self.debug(f"Error in interact.sh: {e}") diff --git a/bbot/modules/github.py b/bbot/modules/github.py index c94bcfcdb4..9fddf0393e 100644 --- a/bbot/modules/github.py +++ b/bbot/modules/github.py @@ -11,19 +11,19 @@ class github(shodan_dns): base_url = "https://api.github.com" - def setup(self): - ret = super().setup() + async def setup(self): + ret = await super().setup() self.headers = {"Authorization": f"token {self.api_key}"} return ret - def ping(self): + async def ping(self): url = f"{self.base_url}/zen" - response = self.helpers.request(url) + response = await self.helpers.request(url) assert getattr(response, "status_code", 0) == 200 - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - for repo_url, raw_urls in self.query(query).items(): + for repo_url, raw_urls in (await self.query(query)).items(): repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", source=event) if repo_event is None: continue @@ -35,33 +35,37 @@ def handle_event(self, event): url_event.scope_distance = repo_event.scope_distance self.emit_event(url_event) - def query(self, query): + async def query(self, query): repos = {} url = f"{self.base_url}/search/code?per_page=100&type=Code&q={self.helpers.quote(query)}&page=" + "{page}" - for r in self.helpers.api_page_iter(url, headers=self.headers, json=False): - if r is None: - continue - status_code = getattr(r, "status_code", 0) - if status_code == 429: - "Github is rate-limiting us (HTTP status: 429)" - break - try: - j = r.json() - except Exception as e: - self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") - continue - items = j.get("items", []) - if not items: - break - for item in items: - htlm_url = item.get("html_url", "") - raw_url = self.raw_url(htlm_url) - repo_url = item.get("repository", {}).get("html_url", "") - if raw_url and repo_url: - try: - repos[repo_url].append(raw_url) - except KeyError: - repos[repo_url] = [raw_url] + agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + try: + async for r in agen: + if r is None: + continue + status_code = getattr(r, "status_code", 0) + if status_code == 429: + "Github is rate-limiting us (HTTP status: 429)" + break + try: + j = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + continue + items = j.get("items", []) + if not items: + break + for item in items: + htlm_url = item.get("html_url", "") + raw_url = self.raw_url(htlm_url) + repo_url = item.get("repository", {}).get("html_url", "") + if raw_url and repo_url: + try: + repos[repo_url].append(raw_url) + except KeyError: + repos[repo_url] = [raw_url] + finally: + agen.aclose() return repos @staticmethod diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py index d469488dbb..0bb0f88e94 100644 --- a/bbot/modules/host_header.py +++ b/bbot/modules/host_header.py @@ -12,14 +12,14 @@ class host_header(BaseModule): deps_apt = ["curl"] - def setup(self): + async def setup(self): self.scanned_hosts = set() self.subdomain_tags = {} if self.scan.config.get("interactsh_disable", False) == False: try: self.interactsh_instance = self.helpers.interactsh() - self.domain = self.interactsh_instance.register(callback=self.interactsh_callback) + self.domain = await self.interactsh_instance.register(callback=self.interactsh_callback) except InteractshError as e: self.warning(f"Interactsh failure: {e}") return False @@ -54,25 +54,26 @@ def interactsh_callback(self, r): # this is likely caused by something trying to resolve the base domain first and can be ignored self.debug("skipping results because subdomain tag was missing") - def finish(self): + async def finish(self): if self.scan.config.get("interactsh_disable", False) == False: + await self.helpers.sleep(5) try: - for r in self.interactsh_instance.poll(): + for r in await self.interactsh_instance.poll(): self.interactsh_callback(r) except InteractshError as e: self.debug(f"Error in interact.sh: {e}") - def cleanup(self): + async def cleanup(self): if self.scan.config.get("interactsh_disable", False) == False: try: - self.interactsh_instance.deregister() + await self.interactsh_instance.deregister() self.debug( f"successfully deregistered interactsh session with correlation_id {self.interactsh_instance.correlation_id}" ) except InteractshError as e: self.warning(f"Interactsh failure: {e}") - def handle_event(self, event): + async def handle_event(self, event): host = f"{event.parsed.scheme}://{event.parsed.netloc}/" host_hash = hash(host) if host_hash in self.scanned_hosts: @@ -100,7 +101,7 @@ def handle_event(self, event): self.debug(f"Performing {technique_description} case") subdomain_tag = self.rand_string(4, digits=False) self.subdomain_tags[subdomain_tag] = (event, technique_description) - output = self.helpers.curl( + output = await self.helpers.curl( url=event.data["url"], headers={"Host": f"{subdomain_tag}.{self.domain}"}, ignore_bbot_global_settings=True, @@ -114,7 +115,7 @@ def handle_event(self, event): self.debug(f"Performing {technique_description} case") subdomain_tag = self.rand_string(4, digits=False) self.subdomain_tags[subdomain_tag] = (event, technique_description) - output = self.helpers.curl( + output = await self.helpers.curl( url=event.data["url"], headers={"Host": f"{subdomain_tag}.{self.domain}"}, path_override=event.data["url"], @@ -126,7 +127,7 @@ def handle_event(self, event): # duplicate host header tolerance technique_description = "duplicate host header tolerance" - output = self.helpers.curl( + output = await self.helpers.curl( url=event.data["url"], # Sending a blank HOST first as a hack to trick curl. This makes it no longer an "internal header", thereby allowing for duplicates # The fact that it's accepting two host headers is rare enough to note on its own, and not too noisy. Having the 3rd header be an interactsh would result in false negatives for the slightly less interesting cases. @@ -167,7 +168,7 @@ def handle_event(self, event): for oh in override_headers_list: override_headers[oh] = f"{subdomain_tag}.{self.domain}" - output = self.helpers.curl( + output = await self.helpers.curl( url=event.data["url"], headers=override_headers, cookies=added_cookies, diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 16ca7294c2..4b110bea5d 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -145,6 +145,6 @@ async def handle_batch(self, *events): # HTTP response self.emit_event(j, "HTTP_RESPONSE", url_event, internal=True) - def cleanup(self): + async def cleanup(self): resume_file = self.helpers.current_dir / "resume.cfg" resume_file.unlink(missing_ok=True) diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index 8aef3d1dde..a5636ed62f 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -19,15 +19,15 @@ class Ipstack(shodan_dns): base_url = "http://api.ipstack.com/" - def ping(self): - r = self.request_with_fail_count(f"{self.base_url}/check?access_key={self.api_key}") + async def ping(self): + r = await self.request_with_fail_count(f"{self.base_url}/check?access_key={self.api_key}") resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content - def handle_event(self, event): + async def handle_event(self, event): try: url = f"{self.base_url}/{event.data}?access_key={self.api_key}" - result = self.request_with_fail_count(url) + result = await self.request_with_fail_count(url) if result: j = result.json() if not j: diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 684267806c..6ef511560c 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -1,7 +1,5 @@ -from threading import Lock - from bbot.modules.base import BaseModule -from bbot.core.errors import NTLMError, RequestException +from bbot.core.errors import NTLMError, RequestError ntlm_discovery_endpoints = [ "", @@ -70,14 +68,13 @@ class ntlm(BaseModule): in_scope_only = True - def setup(self): + async def setup(self): self.processed = set() - self.processed_lock = Lock() self.found = set() self.try_all = self.config.get("try_all", False) return True - def handle_event(self, event): + async def handle_event(self, event): found_hash = hash(f"{event.host}:{event.port}") if found_hash not in self.found: result_FQDN, request_url = self.handle_url(event) @@ -94,7 +91,7 @@ def handle_event(self, event): ) self.emit_event(result_FQDN, "DNS_NAME", source=event) - def filter_event(self, event): + async def filter_event(self, event): if self.try_all: return True if event.type == "HTTP_RESPONSE": @@ -104,7 +101,7 @@ def filter_event(self, event): return True return False - def handle_url(self, event): + async def handle_url(self, event): if event.type == "URL": urls = { event.data, @@ -117,35 +114,33 @@ def handle_url(self, event): for endpoint in ntlm_discovery_endpoints: urls.add(f"{event.parsed.scheme}://{event.parsed.netloc}/{endpoint}") - futures = {} + tasks = [] for url in urls: - future = self.submit_task(self.check_ntlm, url) - futures[future] = url + url_hash = hash(url) + if url_hash in self.processed: + continue + self.processed.add(url_hash) + task = self.helpers.create_task(self.check_ntlm(url)) + tasks.append(task) - for future in self.helpers.as_completed(futures): - url = futures[future] + for task in self.helpers.as_completed(tasks): try: - result = future.result() + result, url = await task if result: - for future in futures: - future.cancel() + self.helpers.cancel_tasks(tasks) return str(result["FQDN"]), url - except RequestException as e: + except RequestError as e: self.warning(str(e)) + # cancel all the tasks if there's an error + self.helpers.cancel_tasks(tasks) + break return None, None - def check_ntlm(self, test_url): - url_hash = hash(test_url) - - with self.processed_lock: - if url_hash in self.processed: - return - self.processed.add(url_hash) - + async def check_ntlm(self, test_url): # use lower timeout value http_timeout = self.config.get("httpx_timeout", 5) - r = self.helpers.request( + r = await self.helpers.request( test_url, headers=NTLM_test_header, raise_error=True, allow_redirects=False, timeout=http_timeout ) ntlm_resp = r.headers.get("WWW-Authenticate", "") @@ -154,6 +149,6 @@ def check_ntlm(self, test_url): try: ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64) if ntlm_resp_decoded: - return ntlm_resp_decoded + return ntlm_resp_decoded, test_url except NTLMError as e: self.verbose(str(e)) diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index 5ffb3d86ae..d55794b74a 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -44,7 +44,7 @@ async def setup(self): ret = await super().setup() return ret - def filter_event(self, event): + async def filter_event(self, event): if event._internal: return False, "event is internal" if event.type not in self.watched_events: @@ -159,7 +159,7 @@ def emit_contents(self): f"use_previous=True was set but no previous asset inventory was found at {self.output_file}" ) - def finish(self): + async def finish(self): self.emit_contents() def _run_hooks(self): diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 1e3449df3b..93fa92f3ed 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,6 +1,4 @@ -import requests -from requests.auth import HTTPBasicAuth -from requests.exceptions import RequestException +from bbot.core.errors import RequestError from bbot.modules.output.base import BaseOutputModule @@ -26,31 +24,39 @@ class HTTP(BaseOutputModule): } async def setup(self): - self.session = requests.Session() - if not self.config.get("url", ""): + self.url = self.config.get("url", "") + self.method = self.config.get("method", "POST") + self.timeout = self.config.get("timeout", 10) + self.headers = {} + bearer = self.config.get("bearer", "") + if bearer: + self.headers["Authorization"] = f"Bearer {bearer}" + username = self.config.get("username", "") + password = self.config.get("password", "") + self.auth = None + if username: + self.auth = (username, password) + if not self.url: self.warning("Must set URL") return False - if not self.config.get("method", ""): + if not self.method: self.warning("Must set HTTP method") return False return True - def handle_event(self, event): - r = requests.Request( - url=self.config.get("url"), - method=self.config.get("method", "POST"), - ) - r.headers["User-Agent"] = self.scan.useragent - r.json = dict(event) - username = self.config.get("username", "") - password = self.config.get("password", "") - if username: - r.auth = HTTPBasicAuth(username, password) - bearer = self.config.get("bearer", "") - if bearer: - r.headers["Authorization"] = f"Bearer {bearer}" - try: - timeout = self.config.get("timeout", 10) - self.session.send(r.prepare(), timeout=timeout) - except RequestException as e: - self.warning(f"Error sending {event}: {e}") + async def handle_event(self, event): + while 1: + try: + await self.helpers.request( + url=self.url, + method=self.method, + auth=self.auth, + headers=self.headers, + json=dict(event), + raise_error=True, + ) + break + except RequestError as e: + self.warning(f"Error sending {event}: {e}, retrying...") + self.trace() + await self.helpers.sleep(1) diff --git a/bbot/modules/report/affiliates.py b/bbot/modules/report/affiliates.py index 34a34abb8f..bb6323664d 100644 --- a/bbot/modules/report/affiliates.py +++ b/bbot/modules/report/affiliates.py @@ -9,14 +9,14 @@ class affiliates(BaseReportModule): scope_distance_modifier = None accept_dupes = True - def setup(self): + async def setup(self): self.affiliates = {} return True - def handle_event(self, event): + async def handle_event(self, event): self.add_affiliate(event) - def report(self): + async def report(self): affiliates = sorted(self.affiliates.items(), key=lambda x: x[-1]["weight"], reverse=True) header = ["Affiliate", "Score", "Count"] table = [] diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index ded5e4ee94..9cefdae204 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -16,18 +16,18 @@ class shodan_dns(crobat): base_url = "https://api.shodan.io" - def setup(self): + async def setup(self): super().setup() return self.require_api_key() - def ping(self): - r = self.request_with_fail_count(f"{self.base_url}/api-info?key={self.api_key}") + async def ping(self): + r = await self.request_with_fail_count(f"{self.base_url}/api-info?key={self.api_key}") resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}" - return self.request_with_fail_count(url) + return await self.request_with_fail_count(url) def parse_results(self, r, query): json = r.json() diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 30d1281450..b49e88a09b 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -7,7 +7,7 @@ from bbot.modules.base import BaseModule from bbot.core.errors import ValidationError -from bbot.core.helpers.threadpool import NamedLock +from bbot.core.helpers.async_helpers import NamedLock class sslcert(BaseModule): diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py index 19386eb6b2..0714bdc8e5 100644 --- a/bbot/modules/viewdns.py +++ b/bbot/modules/viewdns.py @@ -20,27 +20,27 @@ class viewdns(BaseModule): in_scope_only = True _qsize = 1 - def setup(self): + async def setup(self): self.processed = set() self.date_regex = re.compile(r"\d{4}-\d{2}-\d{2}") return True - def filter_event(self, event): + async def filter_event(self, event): _, domain = self.helpers.split_domain(event.data) if hash(domain) in self.processed: return False self.processed.add(hash(domain)) return True - def handle_event(self, event): + async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) - for domain, _ in self.query(query): + for domain, _ in await self.query(query): self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) # todo: registrar? - def query(self, query): + async def query(self, query): url = f"{self.base_url}/reversewhois/?q={query}" - r = self.helpers.request(url) + r = await self.helpers.request(url) status_code = getattr(r, "status_code", 0) if status_code not in (200,): self.verbose(f"Error retrieving reverse whois results (status code: {status_code})") diff --git a/bbot/test/test_step_2/test_modules_basic.py b/bbot/test/test_step_2/test_modules_basic.py index 5c5285ad2c..9925eb1c8b 100644 --- a/bbot/test/test_step_2/test_modules_basic.py +++ b/bbot/test/test_step_2/test_modules_basic.py @@ -12,6 +12,7 @@ async def test_modules_basic( with open(fallback_nameservers, "w") as f: f.write("8.8.8.8\n") + httpx_mock.assert_all_responses_were_requested = False for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"): httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"}) @@ -90,13 +91,21 @@ async def test_modules_basic( scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers patch_commands(scan2) patch_ansible(scan2) - scan2.load_modules() + await scan2.load_modules() scan2.status = "RUNNING" # attributes, descriptions, etc. - for module_name, module in scan2.modules.items(): + for module_name, module in sorted(scan2.modules.items()): # flags assert module._type in ("internal", "output", "scan") + # async stuff + not_async = [] + for func_name in ("setup", "ping", "filter_event", "handle_event", "finish", "report", "cleanup"): + f = getattr(module, func_name) + if not scan2.helpers.is_async_function(f): + log.error(f"{f.__qualname__}() is not async") + not_async.append(f) + assert not any(not_async) # module preloading all_preloaded = module_loader.preloaded() From 8c6e413d95f014e27c8da426929139b39417b8d8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 May 2023 18:07:47 -0400 Subject: [PATCH 024/387] more module tests, sslcert suffering --- bbot/core/helpers/misc.py | 2 +- bbot/core/helpers/modules.py | 3 +- bbot/modules/base.py | 4 +- bbot/modules/bevigil.py | 2 +- bbot/modules/binaryedge.py | 2 +- bbot/modules/bucket_gcp.py | 1 + bbot/modules/output/asset_inventory.py | 5 +- bbot/modules/output/base.py | 4 + bbot/modules/shodan_dns.py | 4 +- bbot/modules/sslcert.py | 117 +++++++++++-------------- bbot/test/bbot_fixtures.py | 19 ---- bbot/test/conftest.py | 19 ++++ bbot/test/pytest.ini | 1 - bbot/test/test.conf | 5 ++ poetry.lock | 9 +- 15 files changed, 95 insertions(+), 102 deletions(-) delete mode 100644 bbot/test/pytest.ini diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 4f5bc3d615..ed8c414e37 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -27,8 +27,8 @@ import cloudcheck as _cloudcheck import tldextract as _tldextract from hashlib import sha1 as hashlib_sha1 -from asyncio import as_completed, create_task, sleep # noqa from urllib.parse import urlparse, quote, unquote, urlunparse # noqa F401 +from asyncio import as_completed, create_task, sleep, wait_for # noqa from .url import * # noqa F401 from . import regexes diff --git a/bbot/core/helpers/modules.py b/bbot/core/helpers/modules.py index b121aca74f..16b2755e2f 100644 --- a/bbot/core/helpers/modules.py +++ b/bbot/core/helpers/modules.py @@ -1,6 +1,7 @@ import ast import sys import importlib +import traceback from pathlib import Path from omegaconf import OmegaConf from contextlib import suppress @@ -41,8 +42,6 @@ def preload(self, module_dir): self._configs[module_file.stem] = config self._preloaded[module_file.stem] = preloaded except Exception: - import traceback - print(f"[CRIT] Error preloading {module_file}\n\n{traceback.format_exc()}") print(f"[CRIT] Error in {module_file.name}") sys.exit(1) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index e359ed7816..49cad93f91 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -158,14 +158,14 @@ async def cleanup(self): """ return - def require_api_key(self): + async def require_api_key(self): """ Use in setup() to ensure the module is configured with an API key """ self.api_key = self.config.get("api_key", "") if self.auth_secret: try: - self.ping() + await self.ping() self.hugesuccess(f"API is ready") return True except Exception as e: diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index a24d6331c1..3bf821e759 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -19,7 +19,7 @@ async def setup(self): self.api_key = self.config.get("api_key", "") self.headers = {"X-Access-Token": self.api_key} self.urls = self.config.get("urls", False) - return super().setup() + return await super().setup() async def ping(self): pass diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py index 497dd03069..637585f9fc 100644 --- a/bbot/modules/binaryedge.py +++ b/bbot/modules/binaryedge.py @@ -17,7 +17,7 @@ class binaryedge(shodan_dns): async def setup(self): self.max_records = self.config.get("max_records", 1000) self.headers = {"X-Key": self.config.get("api_key", "")} - return super().setup() + return await super().setup() async def ping(self): url = f"{self.base_url}/user/subscription" diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py index b7e96d5b1d..d4ac880a9c 100644 --- a/bbot/modules/bucket_gcp.py +++ b/bbot/modules/bucket_gcp.py @@ -50,4 +50,5 @@ async def check_bucket_exists(self, bucket_name, url): response = await self.helpers.request(url) status_code = getattr(response, "status_code", 0) existent_bucket = status_code not in (0, 400, 404) + self.critical(f"{bucket_name}: {url}: {existent_bucket}") return existent_bucket, set(), bucket_name, url diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index d55794b74a..ce1f44e98b 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -56,7 +56,7 @@ async def filter_event(self, event): return True, "" async def handle_event(self, event): - if self.filter_event(event)[0]: + if (await self.filter_event(event))[0]: hostkey = _make_hostkey(event.host, event.resolved_hosts) if hostkey not in self.assets: self.assets[hostkey] = Asset(event.host) @@ -117,8 +117,7 @@ def increment_stat(stat, value): self.log_table(table, table_header, table_name=f"asset-inventory-{header}") if self._file is not None: - with self._report_lock: - self.info(f"Saved asset-inventory output to {self.output_file}") + self.info(f"Saved asset-inventory output to {self.output_file}") def emit_contents(self): if self.use_previous and not self.emitted_contents: diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 5a01b0307f..67880009f5 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -10,10 +10,14 @@ class BaseOutputModule(BaseModule): _stats_exclude = True def _event_precheck(self, event): + # omitted events such as HTTP_RESPONSE etc. if event._omit: return False, "_omit is True" + # forced events like intermediary links in a DNS resolution chain if event._force_output: return True, "_force_output is True" + # internal events like those from speculate, ipneighbor + # or events that are over our report distance if event._internal: return False, "_internal is True" return super()._event_precheck(event) diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 9cefdae204..5323e74a40 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -17,8 +17,8 @@ class shodan_dns(crobat): base_url = "https://api.shodan.io" async def setup(self): - super().setup() - return self.require_api_key() + await super().setup() + return await self.require_api_key() async def ping(self): r = await self.request_with_fail_count(f"{self.base_url}/api-info?key={self.api_key}") diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index b49e88a09b..cc437cc0d0 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -1,8 +1,6 @@ -import select -import socket -import threading -from OpenSSL import SSL -from ssl import PROTOCOL_TLSv1 +import ssl +import asyncio +from OpenSSL import crypto from contextlib import suppress from bbot.modules.base import BaseModule @@ -26,7 +24,7 @@ class sslcert(BaseModule): scope_distance_modifier = 1 _priority = 2 - def setup(self): + async def setup(self): self.timeout = self.config.get("timeout", 5.0) self.skip_non_ssl = self.config.get("skip_non_ssl", True) self.non_ssl_ports = (22, 53, 80) @@ -38,16 +36,15 @@ def setup(self): self.out_of_scope_abort_threshold = 10 self.hosts_visited = set() - self.hosts_visited_lock = threading.Lock() self.ip_lock = NamedLock() return True - def filter_event(self, event): + async def filter_event(self, event): if self.skip_non_ssl and event.port in self.non_ssl_ports: return False, f"Port {event.port} doesn't typically use SSL" return True - def handle_event(self, event): + async def handle_event(self, event): _host = event.host if event.port: port = event.port @@ -58,12 +55,7 @@ def handle_event(self, event): if self.helpers.is_ip(_host): hosts = [_host] else: - hosts = list(self.helpers.resolve(_host)) - - futures = {} - for host in hosts: - future = self.submit_task(self.visit_host, host, port) - futures[future] = host + hosts = list(await self.helpers.resolve(_host)) if event.scope_distance == 0: abort_threshold = self.in_scope_abort_threshold @@ -71,12 +63,17 @@ def handle_event(self, event): else: abort_threshold = self.out_of_scope_abort_threshold log_fn = self.verbose - for future in self.helpers.as_completed(futures): - host = futures[future] - result = future.result() - if not isinstance(result, tuple) or not len(result) == 2: + + tasks = [] + for host in hosts: + task = self.helpers.create_task(self.visit_host(host, port)) + tasks.append(task) + + for task in self.helpers.as_completed(tasks): + result = await task + if not isinstance(result, tuple) or not len(result) == 3: continue - dns_names, emails = result + dns_names, emails, (host, port) = result if len(dns_names) > abort_threshold: netloc = self.helpers.make_netloc(host, port) log_fn( @@ -100,61 +97,51 @@ def on_success_callback(self, event): if source_scope_distance == 0 and event.scope_distance > 0: event.add_tag("affiliate") - def visit_host(self, host, port): + async def visit_host(self, host, port): host = self.helpers.make_ip_type(host) netloc = self.helpers.make_netloc(host, port) host_hash = hash((host, port)) dns_names = [] emails = set() - with self.ip_lock.get_lock(host_hash): - with self.hosts_visited_lock: - if host_hash in self.hosts_visited: - self.debug(f"Already processed {host} on port {port}, skipping") - return [], [] - else: - self.hosts_visited.add(host_hash) - - socket_type = socket.AF_INET - if self.helpers.is_ip(host): - if host.version == 6: - socket_type = socket.AF_INET6 + async with self.ip_lock.lock(host_hash): + if host_hash in self.hosts_visited: + self.debug(f"Already processed {host} on port {port}, skipping") + return [], [], (host, port) + else: + self.hosts_visited.add(host_hash) + host = str(host) + + # Create an SSL context try: - sock = socket.socket(socket_type, socket.SOCK_STREAM) + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE except Exception as e: - self.warning(f"Error creating socket for {netloc}: {e}. Do you have IPv6 disabled?") - return [], [] - sock.settimeout(self.timeout) - try: - context = SSL.Context(PROTOCOL_TLSv1) - except AttributeError as e: - # AttributeError: module 'lib' has no attribute 'SSL_CTX_set_ecdh_auto' self.warning(f"Error creating SSL context: {e}") - return [], [] - self.debug(f"Connecting to {host} on port {port}") - try: - sock.connect((host, port)) - except Exception as e: - self.debug(f"Error connecting to {host} on port {port}: {e}") - return [], [] - connection = SSL.Connection(context, sock) - connection.set_tlsext_host_name(self.helpers.smart_encode(host)) - connection.set_connect_state() + return [], [], (host, port) + + # Connect to the host try: - while 1: - try: - connection.do_handshake() - except SSL.WantReadError: - rd, _, _ = select.select([sock], [], [], sock.gettimeout()) - if not rd: - raise SSL.Error("select timed out") - continue - break + transport, _ = await self.scan._loop.create_connection( + lambda: asyncio.Protocol(), host, port, ssl=ssl_context + ) except Exception as e: - self.debug(f"Error with SSL handshake on {host} port {port}: {e}") - return [], [] - cert = connection.get_peer_certificate() - sock.close() + log_fn = self.warning + if isinstance(e, OSError): + log_fn = self.debug + log_fn(f"Error connecting to {netloc}: {e}") + return [], [], (host, port) + finally: + with suppress(Exception): + transport.close() + + # Get the SSL object + ssl_object = transport.get_extra_info("ssl_object") + + # Get the certificate + der = ssl_object.getpeercert(binary_form=True) + cert = crypto.load_certificate(crypto.FILETYPE_ASN1, der) issuer = cert.get_issuer() if issuer.emailAddress and self.helpers.regexes.email_regex.match(issuer.emailAddress): emails.add(issuer.emailAddress) @@ -166,7 +153,7 @@ def visit_host(self, host, port): with suppress(KeyError): dns_names.remove(common_name) dns_names = [common_name] + list(dns_names) - return dns_names, list(emails) + return dns_names, list(emails), (host, port) @staticmethod def get_cert_sans(cert): diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 5b8d2ffe0e..2b0553475a 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -7,7 +7,6 @@ import tldextract from pathlib import Path from omegaconf import OmegaConf -from pytest_httpserver import HTTPServer import pytest_httpserver from werkzeug.wrappers import Request @@ -337,21 +336,3 @@ def install_all_python_deps(): for module in module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) - - -@pytest.fixture -def bbot_httpserver(): - server = HTTPServer(host="127.0.0.1", port=8888) - server.start() - - yield server - - server.clear() - if server.is_running(): - server.stop() - - # this is to check if the client has made any request where no - # `assert_request` was called on it from the test - - server.check_assertions() - server.clear() diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 85a3f82504..be448d9feb 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,6 +1,7 @@ import shutil import pytest import logging +from pytest_httpserver import HTTPServer @pytest.hookimpl(tryfirst=True, hookwrapper=True) @@ -16,3 +17,21 @@ def pytest_sessionfinish(session, exitstatus): shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield + + +@pytest.fixture +def bbot_httpserver(): + server = HTTPServer(host="127.0.0.1", port=8888) + server.start() + + yield server + + server.clear() + if server.is_running(): + server.stop() + + # this is to check if the client has made any request where no + # `assert_request` was called on it from the test + + server.check_assertions() + server.clear() diff --git a/bbot/test/pytest.ini b/bbot/test/pytest.ini deleted file mode 100644 index 3396baacfc..0000000000 --- a/bbot/test/pytest.ini +++ /dev/null @@ -1 +0,0 @@ -log_level=DEBUG \ No newline at end of file diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 80daae5a53..86dcb526b9 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -41,3 +41,8 @@ excavate: false aggregate: false omit_event_types: [] debug: true +dns_wildcard_ignore: + - blacklanternsecurity.com + - google + - google.com + - example.com \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 146a03dd54..1588260caf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1155,18 +1155,17 @@ files = [ [[package]] name = "pytest" -version = "7.2.2" +version = "7.3.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, - {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, ] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" @@ -1175,7 +1174,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-asyncio" From 55199e28d7cbf5f3f30511fd93c27e104650c480 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 May 2023 18:14:30 -0400 Subject: [PATCH 025/387] restored timeout --- bbot/modules/sslcert.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index cc437cc0d0..947bbe6aa5 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -123,9 +123,12 @@ async def visit_host(self, host, port): # Connect to the host try: - transport, _ = await self.scan._loop.create_connection( + transport, _ = await asyncio.wait_for(self.scan._loop.create_connection( lambda: asyncio.Protocol(), host, port, ssl=ssl_context - ) + ), timeout=self.timeout) + except asyncio.TimeoutError: + self.debug(f"Timed out after {self.timeout} seconds while connecting to {netloc}") + return [], [], (host, port) except Exception as e: log_fn = self.warning if isinstance(e, OSError): From 912697f4b9bd3a7e097c97ddd3bf692b247505ac Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 17 May 2023 18:14:40 -0400 Subject: [PATCH 026/387] blacked --- bbot/modules/sslcert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 947bbe6aa5..8792cecfa9 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -123,9 +123,10 @@ async def visit_host(self, host, port): # Connect to the host try: - transport, _ = await asyncio.wait_for(self.scan._loop.create_connection( - lambda: asyncio.Protocol(), host, port, ssl=ssl_context - ), timeout=self.timeout) + transport, _ = await asyncio.wait_for( + self.scan._loop.create_connection(lambda: asyncio.Protocol(), host, port, ssl=ssl_context), + timeout=self.timeout, + ) except asyncio.TimeoutError: self.debug(f"Timed out after {self.timeout} seconds while connecting to {netloc}") return [], [], (host, port) From e3ba806ececd07baf5de4b4ceeb96025c4b59093 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 18 May 2023 23:18:45 -0400 Subject: [PATCH 027/387] steadily writing module tests --- bbot/core/helpers/diff.py | 4 +- bbot/core/helpers/web.py | 3 + bbot/modules/builtwith.py | 23 ++++-- bbot/modules/censys.py | 158 +++++++++++++----------------------- bbot/modules/crt.py | 2 +- bbot/modules/dnsdumpster.py | 6 +- bbot/test/conftest.py | 5 ++ bbot/test/test.conf | 1 + 8 files changed, 88 insertions(+), 114 deletions(-) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 2f191e0db6..7203071df2 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -47,7 +47,9 @@ async def _baseline(self): if baseline_1 is None or baseline_2 is None: log.debug("HTTP error while establishing baseline, aborting") - raise HttpCompareError("Can't get baseline from source URL") + raise HttpCompareError( + f"Can't get baseline from source URL: {url_1}:{baseline_1} / {url_2}:{baseline_2}" + ) if baseline_1.status_code != baseline_2.status_code: log.debug("Status code not stable during baseline, aborting") raise HttpCompareError("Can't get baseline from source URL") diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 6ed9fe8427..d47b6289f2 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -175,6 +175,9 @@ async def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs): offset = 0 while 1: new_url = url.format(page=page, page_size=page_size, offset=offset) + data = requests_kwargs.get("data", None) + if data is not None: + requests_kwargs["data"] = requests_kwargs["data"].format(page=page, page_size=page_size, offset=offset) result = await self.request(new_url, **requests_kwargs) try: if json: diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index e8139d5dd8..6fabcd2833 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -52,7 +52,7 @@ async def request_redirects(self, query): def parse_domains(self, r, query): """ - This method yields subdomains. + This method returns a set of subdomains. Each subdomain is an "FQDN" that was reported in the "Detailed Technology Profile" page on builtwith.com Parameters @@ -60,20 +60,25 @@ def parse_domains(self, r, query): r (requests Response): The raw requests response from the API query (string): The query used against the API """ + results_set = set() + self.critical(r.content) json = r.json() if json: - for result in json.get("Results", []): - for chunk in result.get("Result", {}).get("Paths", []): - domain = chunk.get("Domain", "") - subdomain = chunk.get("SubDomain", "") - if domain: - if subdomain: - domain = f"{subdomain}.{domain}" - yield domain + results = json.get("Results", []) + if results: + for result in results: + for chunk in result.get("Result", {}).get("Paths", []): + domain = chunk.get("Domain", "") + subdomain = chunk.get("SubDomain", "") + if domain: + if subdomain: + domain = f"{subdomain}.{domain}" + results_set.add(domain) else: error = json.get("Errors", [{}])[0].get("Message", "Unknown Error") if error: self.verbose(f"No results for {query}: {error}") + return results_set def parse_redirects(self, r, query): """ diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py index ed713472d7..283b902e37 100644 --- a/bbot/modules/censys.py +++ b/bbot/modules/censys.py @@ -1,122 +1,80 @@ -from contextlib import suppress - -from censys.common import exceptions -from censys.search import CensysHosts -from censys.search import CensysCertificates - from bbot.modules.shodan_dns import shodan_dns class censys(shodan_dns): + """ + thanks to https://github.com/owasp-amass/amass/blob/master/resources/scripts/cert/censys.ads + """ + watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME", "EMAIL_ADDRESS", "IP_ADDRESS", "OPEN_PORT", "PROTOCOL"] flags = ["subdomain-enum", "email-enum", "passive", "safe"] meta = {"description": "Query the Censys API", "auth_required": True} - options = {"api_id": "", "api_secret": "", "max_records": 1000} - options_desc = { - "api_id": "Censys.io API ID", - "api_secret": "Censys.io API Secret", - "max_records": "Limit results to help prevent exceeding API quota", - } + options = {"api_id": "", "api_secret": ""} + options_desc = {"api_id": "Censys.io API ID", "api_secret": "Censys.io API Secret"} - deps_pip = ["censys~=2.1.9"] + base_url = "https://search.censys.io/api/v1" async def setup(self): - self.max_records = self.config.get("max_records", 1000) self.api_id = self.config.get("api_id", "") self.api_secret = self.config.get("api_secret", "") - self._cert_name_threshold = 20 - with suppress(Exception): - self.hosts = CensysHosts(api_id=self.api_id, api_secret=self.api_secret) - with suppress(Exception): - self.certificates = CensysCertificates(api_id=self.api_id, api_secret=self.api_secret) - return super().setup() + self.auth = (self.api_id, self.api_secret) + return await super().setup() async def ping(self): - quota = self.certificates.quota() - used = int(quota["used"]) - allowance = int(quota["allowance"]) + url = f"{self.base_url}/account" + resp = await self.helpers.request(url, auth=self.auth) + d = resp.json() + assert isinstance(d, dict), f"Invalid response from {url}: {resp}" + quota = d.get("quota", {}) + used = int(quota.get("used", 0)) + allowance = int(quota.get("allowance", 0)) assert used < allowance, "No quota remaining" async def query(self, query): - emails = set() - dns_names = set() - ip_addresses = dict() - try: - # certificates - certificate_query = f"parsed.names: {query}" - certificate_fields = ["parsed.names", "parsed.issuer_dn", "parsed.subject_dn"] - results = await self.scan.run_in_executor( - self.certificates.search, certificate_query, fields=certificate_fields, max_records=self.max_records + results = set() + page = 1 + while 1: + resp = await self.helpers.request( + f"{self.base_url}/search/certificates", + method="POST", + json={ + "query": f"parsed.names: {query}", + "page": page, + "fields": ["parsed.names"], + }, + auth=self.auth, ) - for result in results: - parsed_names = result.get("parsed.names", []) - # helps filter out third-party certs with a lot of garbage names - _filter = lambda x: True - domain = self.helpers.tldextract(query).domain - if len(parsed_names) > self._cert_name_threshold: - _filter = lambda x: domain in str(x.lower()) - parsed_names = list(filter(_filter, parsed_names)) - dns_names.update(set([n.lstrip(".*").rstrip(".").lower() for n in parsed_names])) - emails.update(set(self.helpers.extract_emails(result.get("parsed.issuer_dn", "")))) - emails.update(set(self.helpers.extract_emails(result.get("parsed.subject_dn", "")))) - - # hosts - per_page = 100 - pages = max(1, int(self.max_records / per_page)) - hosts_query = f"services.tls.certificates.leaf_data.names: {query} or services.tls.certificates.leaf_data.subject.email_address: {query}" - hosts_results = await self.scan.run_in_executor( - self.hosts.search, hosts_query, per_page=per_page, pages=pages - ) - for i, page in enumerate(hosts_results): - for result in page: - ip = result.get("ip", "") - if not ip: - continue - ip_addresses[ip] = [] - services = result.get("services", []) - for service in services: - port = service.get("port") - service_name = service.get("service_name", "") - transport_protocol = service.get("transport_protocol", "") - if not port or not transport_protocol: - continue - ip_addresses[ip].append((port, service_name, transport_protocol)) - if self.scan.stopping: - break - - except exceptions.CensysRateLimitExceededException: - self.warning("Exceeded Censys account limits") - except exceptions.CensysException as e: - self.warning(f"Error with API: {e}") - except Exception as e: - self.warning(f"Unknown error: {e}") - - return emails, dns_names, ip_addresses - - async def handle_event(self, event): - query = self.make_query(event) - emails, dns_names, ip_addresses = self.query(query) - for email in emails: - self.emit_event(email, "EMAIL_ADDRESS", source=event) - for dns_name in dns_names: - self.emit_event(dns_name, "DNS_NAME", source=event) - for ip, services in ip_addresses.items(): - ip_event = self.make_event(ip, "IP_ADDRESS", source=event) - if not ip_event: - continue - self.emit_event(ip_event) - for port, service_name, transport_protocol in services: - port_data = self.helpers.make_netloc(ip, port) - port_type = f"OPEN_{transport_protocol.upper()}_PORT" - port_event = self.make_event(port_data, port_type, source=ip_event) - if not port_event: - continue - self.emit_event(port_event) - if service_name: - service_name = str(service_name).upper() - protocol_data = {"host": port_data, "protocol": service_name} - self.emit_event(protocol_data, "PROTOCOL", source=port_event) + page += 1 + + if resp is None: + break + + d = resp.json() + if d is None: + break + elif not isinstance(d, dict): + break + + error = d.get("error", "") + if error: + self.warning(error) + + if resp.status_code < 200 or resp.status_code >= 400: + break + + elif d.get("status") is None or d["status"] != "ok" or len(d.get("results", [])) == 0: + break + + for r in d["results"]: + for v in r["parsed.names"]: + results.add(v.strip(".*").lower()) + + metadata = d.get("metadata", {}) + if metadata.get("page", 0) >= metadata.get("pages", 0): + break + + return results @property def auth_secret(self): diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py index a4410354ce..62d4a37fd6 100644 --- a/bbot/modules/crt.py +++ b/bbot/modules/crt.py @@ -12,7 +12,7 @@ class crt(crobat): async def setup(self): self.cert_ids = set() - return super().setup() + return await super().setup() async def request_url(self, query): params = {"q": f"%.{query}", "output": "json"} diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py index 2d7f69a4b7..cff5b0692f 100644 --- a/bbot/modules/dnsdumpster.py +++ b/bbot/modules/dnsdumpster.py @@ -14,10 +14,10 @@ class dnsdumpster(crobat): base_url = "https://dnsdumpster.com" - def query(self, domain): + async def query(self, domain): ret = [] # first, get the CSRF tokens - res1 = self.request_with_fail_count(self.base_url) + res1 = await self.request_with_fail_count(self.base_url) status_code = getattr(res1, "status_code", 0) if status_code in [429]: self.verbose(f'Too many requests "{status_code}"') @@ -56,7 +56,7 @@ def query(self, domain): # Otherwise, do the needful subdomains = set() - res2 = self.request_with_fail_count( + res2 = await self.request_with_fail_count( f"{self.base_url}/", method="POST", cookies={"csrftoken": csrftoken}, diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index be448d9feb..0aca0ff1bd 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -19,6 +19,11 @@ def pytest_sessionfinish(session, exitstatus): yield +@pytest.fixture +def non_mocked_hosts() -> list: + return ["127.0.0.1"] + + @pytest.fixture def bbot_httpserver(): server = HTTPServer(host="127.0.0.1", port=8888) diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 86dcb526b9..602190edd4 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -36,6 +36,7 @@ keep_scans: 1 agent_url: ws://127.0.0.1:8765 agent_token: test dns_resolution: false +dns_timeout: 1 speculate: false excavate: false aggregate: false From ea76f5619baecd7c7ac07b5b198dcbf9794a7de6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 May 2023 20:24:06 -0400 Subject: [PATCH 028/387] rolling on module tests --- bbot/core/helpers/dns.py | 7 + bbot/modules/dnszonetransfer.py | 7 +- bbot/modules/emailformat.py | 11 +- bbot/modules/fullhunt.py | 7 +- bbot/test/helpers.py | 13 - bbot/test/modules_test_classes.py | 393 ------------------ bbot/test/test.conf | 2 + .../test/test_step_1/module_tests/__init__.py | 12 + bbot/test/test_step_1/module_tests/base.py | 132 ++++++ .../module_tests/test__module__tests.py | 35 ++ .../module_tests/test_module_affiliates.py | 11 + .../module_tests/test_module_aggregate.py | 11 + .../module_tests/test_module_anubisdb.py | 13 + .../module_tests/test_module_asn.py | 239 +++++++++++ .../test_module_asset_inventory.py | 17 + .../module_tests/test_module_azure_tenant.py | 46 ++ .../module_tests/test_module_badsecrets.py | 110 +++++ .../module_tests/test_module_bevigil.py | 24 ++ .../module_tests/test_module_binaryedge.py | 31 ++ .../module_tests/test_module_bucket_aws.py | 93 +++++ .../module_tests/test_module_bucket_azure.py | 13 + .../test_module_bucket_digitalocean.py | 10 + .../test_module_bucket_firebase.py | 13 + .../module_tests/test_module_bucket_gcp.py | 27 ++ .../module_tests/test_module_builtwith.py | 110 +++++ .../module_tests/test_module_bypass403.py | 50 +++ .../module_tests/test_module_c99.py | 25 ++ .../module_tests/test_module_censys.py | 41 ++ .../module_tests/test_module_certspotter.py | 14 + .../module_tests/test_module_crobat.py | 12 + .../module_tests/test_module_crt.py | 15 + .../module_tests/test_module_csv.py | 8 + .../module_tests/test_module_dnscommonsrv.py | 19 + .../module_tests/test_module_dnsdumpster.py | 18 + .../test_module_dnszonetransfer.py | 56 +++ .../module_tests/test_module_emailformat.py | 12 + .../module_tests/test_module_excavate.py | 141 +++++++ .../module_tests/test_module_ffuf.py | 45 ++ .../test_module_ffuf_shortnames.py | 208 +++++++++ .../module_tests/test_module_fingerprintx.py | 14 + .../module_tests/test_module_fullhunt.py | 48 +++ .../module_tests/test_module_generic_ssrf.py | 0 .../module_tests/test_module_github.py | 0 .../module_tests/test_module_gowitness.py | 0 .../module_tests/test_module_hackertarget.py | 0 .../module_tests/test_module_host_header.py | 0 .../module_tests/test_module_http.py | 0 .../module_tests/test_module_httpx.py | 0 .../module_tests/test_module_human.py | 0 .../module_tests/test_module_hunt.py | 0 .../module_tests/test_module_hunterio.py | 0 .../test_module_iis_shortnames.py | 0 .../module_tests/test_module_ipneighbor.py | 0 .../module_tests/test_module_ipstack.py | 0 .../module_tests/test_module_json.py | 0 .../module_tests/test_module_leakix.py | 0 .../module_tests/test_module_masscan.py | 0 .../module_tests/test_module_massdns.py | 0 .../module_tests/test_module_naabu.py | 0 .../module_tests/test_module_neo4j.py | 0 .../module_tests/test_module_ntlm.py | 0 .../module_tests/test_module_nuclei.py | 0 .../module_tests/test_module_otx.py | 0 .../test_module_paramminer_cookies.py | 0 .../test_module_paramminer_getparams.py | 0 .../test_module_paramminer_headers.py | 0 .../module_tests/test_module_passivetotal.py | 0 .../module_tests/test_module_pgp.py | 0 .../module_tests/test_module_python.py | 0 .../module_tests/test_module_rapiddns.py | 0 .../module_tests/test_module_riddler.py | 0 .../module_tests/test_module_robots.py | 0 .../module_tests/test_module_secretsdb.py | 0 .../test_module_securitytrails.py | 0 .../module_tests/test_module_services.py | 0 .../module_tests/test_module_shodan_dns.py | 0 .../module_tests/test_module_skymem.py | 0 .../module_tests/test_module_smuggler.py | 0 .../module_tests/test_module_social.py | 0 .../module_tests/test_module_speculate.py | 0 .../module_tests/test_module_sslcert.py | 8 + .../test_module_subdomain_hijack.py | 0 .../module_tests/test_module_sublist3r.py | 0 .../module_tests/test_module_telerik.py | 0 .../module_tests/test_module_threatminer.py | 0 .../test_module_url_manipulation.py | 0 .../module_tests/test_module_urlscan.py | 0 .../module_tests/test_module_vhost.py | 0 .../module_tests/test_module_viewdns.py | 0 .../module_tests/test_module_virustotal.py | 0 .../module_tests/test_module_wafw00f.py | 0 .../module_tests/test_module_wappalyzer.py | 0 .../module_tests/test_module_wayback.py | 0 .../module_tests/test_module_web_report.py | 0 .../module_tests/test_module_websocket.py | 0 .../module_tests/test_module_zoomeye.py | 0 96 files changed, 1703 insertions(+), 418 deletions(-) create mode 100644 bbot/test/test_step_1/module_tests/__init__.py create mode 100644 bbot/test/test_step_1/module_tests/base.py create mode 100644 bbot/test/test_step_1/module_tests/test__module__tests.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_affiliates.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_aggregate.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_anubisdb.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_asn.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_asset_inventory.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_azure_tenant.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_badsecrets.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bevigil.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_binaryedge.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bucket_aws.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bucket_azure.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bucket_digitalocean.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bucket_firebase.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bucket_gcp.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_builtwith.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_bypass403.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_c99.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_censys.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_certspotter.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_crobat.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_crt.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_csv.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_dnsdumpster.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_emailformat.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_excavate.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_ffuf.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_ffuf_shortnames.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_fingerprintx.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_fullhunt.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_github.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_gowitness.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_hackertarget.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_host_header.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_http.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_httpx.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_human.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_hunt.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_hunterio.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_ipneighbor.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_ipstack.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_json.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_leakix.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_masscan.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_massdns.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_naabu.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_neo4j.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_ntlm.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_nuclei.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_otx.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_passivetotal.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_pgp.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_python.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_rapiddns.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_riddler.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_robots.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_secretsdb.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_securitytrails.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_services.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_shodan_dns.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_skymem.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_smuggler.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_social.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_speculate.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_sslcert.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_sublist3r.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_telerik.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_threatminer.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_url_manipulation.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_urlscan.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_vhost.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_viewdns.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_virustotal.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_wafw00f.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_wappalyzer.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_wayback.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_web_report.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_websocket.py create mode 100644 bbot/test/test_step_1/module_tests/test_module_zoomeye.py diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 4f4b651199..3c10bcc48d 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -431,6 +431,13 @@ async def is_wildcard(self, query, ips=None, rdtype=None): Note that is_wildcard can be True, False, or None (indicating that wildcard detection was inconclusive) """ result = {} + + # skip check if the query's parent domain is excluded in the config + for d in self.wildcard_ignore: + if self.parent_helper.host_in_host(query, d): + log.debug(f"Skipping wildcard detection on {query} because it is excluded in the config") + return {} + if rdtype is None: rdtype = "ANY" diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index 4a3fb5a58b..52a9b9634d 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -1,5 +1,6 @@ import dns.zone import dns.query +from functools import partial from bbot.modules.base import BaseModule @@ -36,10 +37,12 @@ async def handle_event(self, event): break try: self.debug(f"Attempting zone transfer against {nameserver} for domain {domain}") - xfr_answer = await dns.asyncquery.xfr(nameserver, domain, timeout=self.timeout, lifetime=self.timeout) + xfr_fn = partial(dns.query.xfr, timeout=self.timeout, lifetime=self.timeout) + xfr_answer = await self.scan.run_in_executor(xfr_fn, nameserver, domain) zone = dns.zone.from_xfr(xfr_answer) except Exception as e: - self.debug(f"Error retrieving zone: {e}") + self.verbose(f"Error retrieving zone: {e}") + self.trace() continue self.hugesuccess(f"Successful zone transfer against {nameserver} for domain {domain}!") finding_description = f"Successful DNS zone transfer against {nameserver} for {domain}" diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py index fa47f23cf5..f28e19f9d9 100644 --- a/bbot/modules/emailformat.py +++ b/bbot/modules/emailformat.py @@ -10,16 +10,13 @@ class emailformat(viewdns): base_url = "https://www.email-format.com" - def extract_emails(self, content): - yield from self.helpers.regexes.email_regex.findall(content) - - def handle_event(self, event): + async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) url = f"{self.base_url}/d/{self.helpers.quote(query)}/" - r = self.request_with_fail_count(url) + r = await self.request_with_fail_count(url) + self.hugesuccess(r.content) if not r: return - for email in self.extract_emails(r.text): - email = email.lower() + for email in self.helpers.extract_emails(r.text): if email.endswith(query): self.emit_event(email, "EMAIL_ADDRESS", source=event) diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 4d6e740b09..8bc5d23265 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -14,17 +14,18 @@ class fullhunt(shodan_dns): async def setup(self): self.api_key = self.config.get("api_key", "") self.headers = {"x-api-key": self.api_key} - return super().setup() + return await super().setup() async def ping(self): url = f"{self.base_url}/auth/status" - j = await self.request_with_fail_count(url, headers=self.headers).json() + j = (await self.request_with_fail_count(url, headers=self.headers)).json() remaining = j["user_credits"]["remaining_credits"] assert remaining > 0, "No credits remaining" async def request_url(self, query): url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains" - return await self.request_with_fail_count(url, headers=self.headers) + response = await self.request_with_fail_count(url, headers=self.headers) + return response def parse_results(self, r, query): return r.json().get("hosts", []) diff --git a/bbot/test/helpers.py b/bbot/test/helpers.py index 0cfafafc99..a4aa453c9a 100644 --- a/bbot/test/helpers.py +++ b/bbot/test/helpers.py @@ -91,16 +91,3 @@ def set_expect_requests(self, expect_args={}, respond_args={}): def _after_scan_prep(self): self.mock_args() - - -def tempwordlist(content): - tmp_path = "/tmp/.bbot_test/" - from bbot.core.helpers.misc import rand_string, mkdir - - mkdir(tmp_path) - filename = f"{tmp_path}{rand_string(8)}" - with open(filename, "w", errors="ignore") as f: - for c in content: - line = f"{c}\n" - f.write(line) - return filename diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index 30088c65cf..a4a7665ebf 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -73,151 +73,6 @@ def check_events(self, events): assert technology, "No TECHNOLOGY emitted" -class Excavate(HttpxMockHelper): - additional_modules = ["httpx"] - targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/links.html"] - - config_overrides = {"web_spider_distance": 1, "web_spider_depth": 1} - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 404 - - def mock_args(self): - response_data = """ - ftp://ftp.test.notreal - \\nhttps://www1.test.notreal - \\x3dhttps://www2.test.notreal - %0ahttps://www3.test.notreal - \\u000ahttps://www4.test.notreal - \nwww5.test.notreal - \\x3dwww6.test.notreal - %0awww7.test.notreal - \\u000awww8.test.notreal - - """ - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": response_data} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # verify relatives path a-tag parsing is working correctly - - expect_args = {"method": "GET", "uri": "/subdir/links.html"} - respond_args = {"response_data": ""} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/relative.html"} - respond_args = {"response_data": ""} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - event_data = [e.data for e in events] - assert "https://www1.test.notreal/" in event_data - assert "https://www2.test.notreal/" in event_data - assert "https://www3.test.notreal/" in event_data - assert "https://www4.test.notreal/" in event_data - assert "www1.test.notreal" in event_data - assert "www2.test.notreal" in event_data - assert "www3.test.notreal" in event_data - assert "www4.test.notreal" in event_data - assert "www5.test.notreal" in event_data - assert "www6.test.notreal" in event_data - assert "www7.test.notreal" in event_data - assert "www8.test.notreal" in event_data - assert "http://www9.test.notreal/" in event_data - - assert "nhttps://www1.test.notreal/" not in event_data - assert "x3dhttps://www2.test.notreal/" not in event_data - assert "a2https://www3.test.notreal/" not in event_data - assert "uac20https://www4.test.notreal/" not in event_data - assert "nwww5.test.notreal" not in event_data - assert "x3dwww6.test.notreal" not in event_data - assert "a2www7.test.notreal" not in event_data - assert "uac20www8.test.notreal" not in event_data - - assert any( - e.type == "FINDING" and e.data.get("description", "") == "Non-HTTP URI: ftp://ftp.test.notreal" - for e in events - ) - assert any( - e.type == "PROTOCOL" - and e.data.get("protocol", "") == "FTP" - and e.data.get("host", "") == "ftp.test.notreal" - for e in events - ) - - assert any( - e.type == "URL_UNVERIFIED" - and e.data == "http://127.0.0.1:8888/relative.html" - and "spider-danger" not in e.tags - for e in events - ) - - assert any( - e.type == "URL_UNVERIFIED" - and e.data == "http://127.0.0.1:8888/2/depth2.html" - and "spider-danger" in e.tags - for e in events - ) - - assert any( - e.type == "URL_UNVERIFIED" - and e.data == "http://127.0.0.1:8888/distance2.html" - and "spider-danger" in e.tags - for e in events - ) - - -class Excavate_relativelinks(HttpxMockHelper): - additional_modules = ["httpx"] - targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/"] - config_overrides = {"web_spider_distance": 1, "web_spider_depth": 1} - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 404 - - def mock_args(self): - # root relative - expect_args = {"method": "GET", "uri": "/rootrelative.html"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # page relative - expect_args = {"method": "GET", "uri": "/subdir/pagerelative.html"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/subdir/"} - respond_args = { - "response_data": "root relativepage relative" - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - root_relative_detection = False - page_relative_detection = False - root_page_confusion_1 = False - root_page_confusion_2 = False - - for e in events: - if e.type == "URL_UNVERIFIED": - # these cases represent the desired behavior for parsing relative links - if e.data == "http://127.0.0.1:8888/rootrelative.html": - root_relative_detection = True - if e.data == "http://127.0.0.1:8888/subdir/pagerelative.html": - page_relative_detection = True - - # these cases indicates that excavate parsed the relative links incorrectly - if e.data == "http://127.0.0.1:8888/pagerelative.html": - root_page_confusion_1 = True - if e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": - root_page_confusion_2 = True - - assert root_relative_detection, "Failed to properly excavate root-relative URL" - assert page_relative_detection, "Failed to properly excavate page-relative URL" - assert not root_page_confusion_1, "Incorrectly detected page-relative URL" - assert not root_page_confusion_2, "Incorrectly detected root-relative URL" - - class Subdomain_Hijack(HttpxMockHelper): additional_modules = ["httpx", "excavate"] @@ -1162,52 +1017,6 @@ def check_events(self, events): assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events) -class Ffuf(HttpxMockHelper): - test_wordlist = ["11111111", "admin", "junkword1", "zzzjunkword2"] - config_overrides = { - "modules": { - "ffuf": { - "wordlist": tempwordlist(test_wordlist), - } - } - } - - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/admin"} - respond_args = {"response_data": "alive admin page"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "URL_UNVERIFIED" and "admin" in e.data for e in events) - assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) - - -class Ffuf_extensions(HttpxMockHelper): - test_wordlist = ["11111111", "console", "junkword1", "zzzjunkword2"] - config_overrides = {"modules": {"ffuf": {"wordlist": tempwordlist(test_wordlist), "extensions": "php"}}} - - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/console.php"} - respond_args = {"response_data": "alive admin page"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "URL_UNVERIFIED" and "console" in e.data for e in events) - assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) - - class Vhost(HttpxMockHelper): targets = ["http://localhost:8888", "secret.localhost"] @@ -1274,208 +1083,6 @@ def check_events(self, events): assert wordcloud_detection -class Ffuf_shortnames(HttpxMockHelper): - test_wordlist = ["11111111", "administrator", "portal", "console", "junkword1", "zzzjunkword2", "directory"] - config_overrides = { - "modules": { - "ffuf_shortnames": { - "find_common_prefixes": True, - "find_common_prefixes": True, - "wordlist": tempwordlist(test_wordlist), - } - } - } - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 404 - - seed_events = [] - parent_event = self.scan.make_event( - "http://127.0.0.1:8888/", "URL", self.scan.root_event, module="httpx", tags=["status-200", "distance-0"] - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ADMINI~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ADM_PO~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ABCZZZ~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ABCXXX~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ABCYYY~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ABCCON~1.ASP", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/DIRECT~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/ADM_DI~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/XYZDIR~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/XYZAAA~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/XYZBBB~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/XYZCCC~1", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-directory"], - ) - ) - seed_events.append( - self.scan.make_event( - "http://127.0.0.1:8888/SHORT~1.PL", - "URL_HINT", - parent_event, - module="iis_shortnames", - tags=["shortname-file"], - ) - ) - self.scan.target._events["http://127.0.0.1:8888"] = seed_events - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/administrator.aspx"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/adm_portal.aspx"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/abcconsole.aspx"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/directory/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/adm_directory/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/xyzdirectory/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/short.pl"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - basic_detection = False - directory_detection = False - prefix_detection = False - delimeter_detection = False - directory_delimeter_detection = False - prefix_delimeter_detection = False - short_extensions_detection = False - - for e in events: - if e.type == "URL_UNVERIFIED": - if e.data == "http://127.0.0.1:8888/administrator.aspx": - basic_detection = True - if e.data == "http://127.0.0.1:8888/directory/": - directory_detection = True - if e.data == "http://127.0.0.1:8888/adm_portal.aspx": - prefix_detection = True - if e.data == "http://127.0.0.1:8888/abcconsole.aspx": - delimeter_detection = True - if e.data == "http://127.0.0.1:8888/abcconsole.aspx": - directory_delimeter_detection = True - if e.data == "http://127.0.0.1:8888/xyzdirectory/": - prefix_delimeter_detection = True - if e.data == "http://127.0.0.1:8888/short.pl": - short_extensions_detection = True - - assert basic_detection - assert directory_detection - assert prefix_detection - assert delimeter_detection - assert directory_delimeter_detection - assert prefix_delimeter_detection - assert short_extensions_detection - - class Iis_shortnames(HttpxMockHelper): additional_modules = ["httpx"] diff --git a/bbot/test/test.conf b/bbot/test/test.conf index 602190edd4..9444e1d14c 100644 --- a/bbot/test/test.conf +++ b/bbot/test/test.conf @@ -44,6 +44,8 @@ omit_event_types: [] debug: true dns_wildcard_ignore: - blacklanternsecurity.com + - fakedomain + - notreal - google - google.com - example.com \ No newline at end of file diff --git a/bbot/test/test_step_1/module_tests/__init__.py b/bbot/test/test_step_1/module_tests/__init__.py new file mode 100644 index 0000000000..0b098d46a2 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/__init__.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from bbot.modules import module_loader + +parent_dir = Path(__file__).parent + +module_test_files = list(parent_dir.glob("test_module_*.py")) +module_test_files = [m.name.split("test_module_")[-1].split(".")[0] for m in module_test_files] + +for module_name in module_loader.preloaded(): + module_name = module_name.lower() + assert module_name in module_test_files, f'No test file found for module "{module_name}"' diff --git a/bbot/test/test_step_1/module_tests/base.py b/bbot/test/test_step_1/module_tests/base.py new file mode 100644 index 0000000000..2f5f935591 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/base.py @@ -0,0 +1,132 @@ +import pytest +import pytest_asyncio +import logging +from abc import abstractmethod +from omegaconf import OmegaConf + +from bbot.scanner import Scanner +from bbot.modules import module_loader +from bbot.core.helpers.misc import rand_string +from ...bbot_fixtures import test_config + +log = logging.getLogger("bbot.test.modules") + + +def tempwordlist(content): + tmp_path = "/tmp/.bbot_test/" + from bbot.core.helpers.misc import rand_string, mkdir + + mkdir(tmp_path) + filename = f"{tmp_path}{rand_string(8)}" + with open(filename, "w", errors="ignore") as f: + for c in content: + line = f"{c}\n" + f.write(line) + return filename + + +class TestClass: + @pytest_asyncio.fixture + async def my_fixture(self, bbot_httpserver): + yield bbot_httpserver + + @pytest.mark.asyncio + async def test_asdf(self, my_fixture): + log.critical(my_fixture) + + +class ModuleTestBase: + targets = ["blacklanternsecurity.com"] + scan_name = None + blacklist = None + whitelist = None + module_name = None + config_overrides = {} + modules_overrides = [] + + class ModuleTest: + def __init__(self, module_test_base, httpx_mock, httpserver, monkeypatch): + self.name = module_test_base.name + self.config = OmegaConf.merge(test_config, OmegaConf.create(module_test_base.config_overrides)) + + self.httpx_mock = httpx_mock + self.httpserver = httpserver + self.monkeypatch = monkeypatch + + # handle output, internal module types + preloaded = module_loader.preloaded() + output_modules = None + modules = list(module_test_base.modules) + output_modules = [] + for module in list(modules): + module_type = preloaded[module]["type"] + if module_type in ("internal", "output"): + modules.remove(module) + if module_type == "output": + output_modules.append(module) + elif module_type == "internal": + self.config = OmegaConf.merge(self.config, {module: True}) + if not output_modules: + output_modules = None + + self.scan = Scanner( + *module_test_base.targets, + modules=modules, + output_modules=output_modules, + name=module_test_base._scan_name, + config=self.config, + whitelist=module_test_base.whitelist, + blacklist=module_test_base.blacklist, + ) + self.events = [] + self.log = logging.getLogger(f"bbot.test.{module_test_base.name}") + + def set_expect_requests(self, expect_args={}, respond_args={}): + if "uri" not in expect_args: + expect_args["uri"] = "/" + self.httpserver.expect_request(**expect_args).respond_with_data(**respond_args) + + @property + def module(self): + return self.scan.modules[self.name] + + @pytest_asyncio.fixture + async def module_test(self, httpx_mock, bbot_httpserver, monkeypatch): + module_test = self.ModuleTest(self, httpx_mock, bbot_httpserver, monkeypatch) + self.setup_before_prep(module_test) + await module_test.scan.prep() + self.setup_after_prep(module_test) + module_test.events = [e async for e in module_test.scan.start()] + yield module_test + + @pytest.mark.asyncio + async def test_module_run(self, module_test): + self.check(module_test, module_test.events) + + @abstractmethod + def check(self, module_test, events): + raise NotImplementedError + + @property + def name(self): + if self.module_name is not None: + return self.module_name + return self.__class__.__name__.split("Test")[-1].lower() + + @property + def _scan_name(self): + if self.scan_name: + return self.scan_name + return f"{self.__class__.__name__.lower()}_test_{rand_string()}" + + @property + def modules(self): + if self.modules_overrides: + return self.modules_overrides + return [self.name] + + def setup_before_prep(self, module_test): + pass + + def setup_after_prep(self, module_test): + pass diff --git a/bbot/test/test_step_1/module_tests/test__module__tests.py b/bbot/test/test_step_1/module_tests/test__module__tests.py new file mode 100644 index 0000000000..4fbc60aa8a --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test__module__tests.py @@ -0,0 +1,35 @@ +import logging +import importlib +from pathlib import Path + +from .base import ModuleTestBase +from bbot.modules import module_loader + +log = logging.getLogger("bbot.test.modules") + +parent_dir = Path(__file__).parent + +_module_test_files = list(parent_dir.glob("test_module_*.py")) +_module_test_files.sort(key=lambda p: p.name) +module_test_files = [m.name.split("test_module_")[-1].split(".")[0] for m in _module_test_files] + + +def test__module__tests(): + # make sure each module has a .py file + for module_name in module_loader.preloaded(): + module_name = module_name.lower() + assert module_name in module_test_files, f'No test file found for module "{module_name}"' + + # make sure each test file has a test class + for file in _module_test_files: + module_name = file.stem + import_path = f"bbot.test.test_step_1.module_tests.{module_name}" + module_test_variables = importlib.import_module(import_path, "bbot") + module_pass = False + for var_name in dir(module_test_variables): + if var_name.startswith("Test"): + test_class = getattr(module_test_variables, var_name) + if ModuleTestBase in getattr(test_class, "__mro__", ()): + module_pass = True + break + assert module_pass, f"Couldn't find a test class for {module_name} in {file}" diff --git a/bbot/test/test_step_1/module_tests/test_module_affiliates.py b/bbot/test/test_step_1/module_tests/test_module_affiliates.py new file mode 100644 index 0000000000..18737cb831 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_affiliates.py @@ -0,0 +1,11 @@ +from .base import ModuleTestBase + + +class TestAffiliates(ModuleTestBase): + targets = ["8.8.8.8"] + config_overrides = {"dns_resolution": True} + + def check(self, module_test, events): + filename = next(module_test.scan.home.glob("affiliates-table*.txt")) + with open(filename) as f: + assert "zdns.google" in f.read() diff --git a/bbot/test/test_step_1/module_tests/test_module_aggregate.py b/bbot/test/test_step_1/module_tests/test_module_aggregate.py new file mode 100644 index 0000000000..b3d72c57e9 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_aggregate.py @@ -0,0 +1,11 @@ +from .base import ModuleTestBase + + +class TestAggregate(ModuleTestBase): + config_overrides = {"dns_resolution": True} + + def check(self, module_test, events): + module_test.log.critical(events) + filename = next(module_test.scan.home.glob("scan-stats-table*.txt")) + with open(filename) as f: + assert "| A " in f.read() diff --git a/bbot/test/test_step_1/module_tests/test_module_anubisdb.py b/bbot/test/test_step_1/module_tests/test_module_anubisdb.py new file mode 100644 index 0000000000..d4a7168d89 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_anubisdb.py @@ -0,0 +1,13 @@ +from .base import ModuleTestBase + + +class TestAnubisdb(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.module.abort_if = lambda e: False + module_test.httpx_mock.add_response( + url=f"https://jldc.me/anubis/subdomains/blacklanternsecurity.com", + json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_asn.py b/bbot/test/test_step_1/module_tests/test_module_asn.py new file mode 100644 index 0000000000..b027dfc04a --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_asn.py @@ -0,0 +1,239 @@ +from .base import ModuleTestBase + + +class TestASNBGPView(ModuleTestBase): + targets = ["8.8.8.8"] + module_name = "asn" + config_overrides = {"scope_report_distance": 2} + + response_get_asn_bgpview = { + "status": "ok", + "status_message": "Query was successful", + "data": { + "ip": "8.8.8.8", + "ptr_record": "dns.google", + "prefixes": [ + { + "prefix": "8.8.8.0/24", + "ip": "8.8.8.0", + "cidr": 24, + "asn": {"asn": 15169, "name": "GOOGLE", "description": "Google LLC", "country_code": "US"}, + "name": "LVLT-GOGL-8-8-8", + "description": "Google LLC", + "country_code": "US", + } + ], + "rir_allocation": { + "rir_name": "ARIN", + "country_code": None, + "ip": "8.0.0.0", + "cidr": 9, + "prefix": "8.0.0.0/9", + "date_allocated": "1992-12-01 00:00:00", + "allocation_status": "allocated", + }, + "iana_assignment": { + "assignment_status": "legacy", + "description": "Administered by ARIN", + "whois_server": "whois.arin.net", + "date_assigned": None, + }, + "maxmind": {"country_code": None, "city": None}, + }, + "@meta": {"time_zone": "UTC", "api_version": 1, "execution_time": "567.18 ms"}, + } + response_get_emails_bgpview = { + "status": "ok", + "status_message": "Query was successful", + "data": { + "asn": 15169, + "name": "GOOGLE", + "description_short": "Google LLC", + "description_full": ["Google LLC"], + "country_code": "US", + "website": "https://about.google/intl/en/", + "email_contacts": ["network-abuse@google.com", "arin-contact@google.com"], + "abuse_contacts": ["network-abuse@google.com"], + "looking_glass": None, + "traffic_estimation": None, + "traffic_ratio": "Mostly Outbound", + "owner_address": ["1600 Amphitheatre Parkway", "Mountain View", "CA", "94043", "US"], + "rir_allocation": { + "rir_name": "ARIN", + "country_code": "US", + "date_allocated": "2000-03-30 00:00:00", + "allocation_status": "assigned", + }, + "iana_assignment": { + "assignment_status": None, + "description": None, + "whois_server": None, + "date_assigned": None, + }, + "date_updated": "2023-02-07 06:39:11", + }, + "@meta": {"time_zone": "UTC", "api_version": 1, "execution_time": "56.55 ms"}, + } + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.bgpview.io/ip/8.8.8.8", json=self.response_get_asn_bgpview + ) + module_test.httpx_mock.add_response( + url="https://api.bgpview.io/asn/15169", json=self.response_get_emails_bgpview + ) + module_test.module.sources = ["bgpview"] + + def check(self, module_test, events): + assert any(e.type == "ASN" for e in events) + assert any(e.type == "EMAIL_ADDRESS" for e in events) + + +class TestASNRipe(ModuleTestBase): + targets = ["8.8.8.8"] + module_name = "asn" + config_overrides = {"scope_report_distance": 2} + + response_get_asn_ripe = { + "messages": [], + "see_also": [], + "version": "1.1", + "data_call_name": "network-info", + "data_call_status": "supported", + "cached": False, + "data": {"asns": ["15169"], "prefix": "8.8.8.0/24"}, + "query_id": "20230217212133-f278ff23-d940-4634-8115-a64dee06997b", + "process_time": 5, + "server_id": "app139", + "build_version": "live.2023.2.1.142", + "status": "ok", + "status_code": 200, + "time": "2023-02-17T21:21:33.428469", + } + response_get_asn_metadata_ripe = { + "messages": [], + "see_also": [], + "version": "4.1", + "data_call_name": "whois", + "data_call_status": "supported - connecting to ursa", + "cached": False, + "data": { + "records": [ + [ + {"key": "ASNumber", "value": "15169", "details_link": None}, + {"key": "ASName", "value": "GOOGLE", "details_link": None}, + {"key": "ASHandle", "value": "15169", "details_link": "https://stat.ripe.net/AS15169"}, + {"key": "RegDate", "value": "2000-03-30", "details_link": None}, + { + "key": "Ref", + "value": "https://rdap.arin.net/registry/autnum/15169", + "details_link": "https://rdap.arin.net/registry/autnum/15169", + }, + {"key": "source", "value": "ARIN", "details_link": None}, + ], + [ + {"key": "OrgAbuseHandle", "value": "ABUSE5250-ARIN", "details_link": None}, + {"key": "OrgAbuseName", "value": "Abuse", "details_link": None}, + {"key": "OrgAbusePhone", "value": "+1-650-253-0000", "details_link": None}, + { + "key": "OrgAbuseEmail", + "value": "network-abuse@google.com", + "details_link": "mailto:network-abuse@google.com", + }, + { + "key": "OrgAbuseRef", + "value": "https://rdap.arin.net/registry/entity/ABUSE5250-ARIN", + "details_link": "https://rdap.arin.net/registry/entity/ABUSE5250-ARIN", + }, + {"key": "source", "value": "ARIN", "details_link": None}, + ], + [ + {"key": "OrgName", "value": "Google LLC", "details_link": None}, + {"key": "OrgId", "value": "GOGL", "details_link": None}, + {"key": "Address", "value": "1600 Amphitheatre Parkway", "details_link": None}, + {"key": "City", "value": "Mountain View", "details_link": None}, + {"key": "StateProv", "value": "CA", "details_link": None}, + {"key": "PostalCode", "value": "94043", "details_link": None}, + {"key": "Country", "value": "US", "details_link": None}, + {"key": "RegDate", "value": "2000-03-30", "details_link": None}, + { + "key": "Comment", + "value": "Please note that the recommended way to file abuse complaints are located in the following links.", + "details_link": None, + }, + { + "key": "Comment", + "value": "To report abuse and illegal activity: https://www.google.com/contact/", + "details_link": None, + }, + { + "key": "Comment", + "value": "For legal requests: http://support.google.com/legal", + "details_link": None, + }, + {"key": "Comment", "value": "Regards,", "details_link": None}, + {"key": "Comment", "value": "The Google Team", "details_link": None}, + { + "key": "Ref", + "value": "https://rdap.arin.net/registry/entity/GOGL", + "details_link": "https://rdap.arin.net/registry/entity/GOGL", + }, + {"key": "source", "value": "ARIN", "details_link": None}, + ], + [ + {"key": "OrgTechHandle", "value": "ZG39-ARIN", "details_link": None}, + {"key": "OrgTechName", "value": "Google LLC", "details_link": None}, + {"key": "OrgTechPhone", "value": "+1-650-253-0000", "details_link": None}, + { + "key": "OrgTechEmail", + "value": "arin-contact@google.com", + "details_link": "mailto:arin-contact@google.com", + }, + { + "key": "OrgTechRef", + "value": "https://rdap.arin.net/registry/entity/ZG39-ARIN", + "details_link": "https://rdap.arin.net/registry/entity/ZG39-ARIN", + }, + {"key": "source", "value": "ARIN", "details_link": None}, + ], + [ + {"key": "RTechHandle", "value": "ZG39-ARIN", "details_link": None}, + {"key": "RTechName", "value": "Google LLC", "details_link": None}, + {"key": "RTechPhone", "value": "+1-650-253-0000", "details_link": None}, + {"key": "RTechEmail", "value": "arin-contact@google.com", "details_link": None}, + { + "key": "RTechRef", + "value": "https://rdap.arin.net/registry/entity/ZG39-ARIN", + "details_link": None, + }, + {"key": "source", "value": "ARIN", "details_link": None}, + ], + ], + "irr_records": [], + "authorities": ["arin"], + "resource": "15169", + "query_time": "2023-02-17T21:25:00", + }, + "query_id": "20230217212529-75f57efd-59f4-473f-8bdd-803062e94290", + "process_time": 268, + "server_id": "app143", + "build_version": "live.2023.2.1.142", + "status": "ok", + "status_code": 200, + "time": "2023-02-17T21:25:29.417812", + } + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://stat.ripe.net/data/network-info/data.json?resource=8.8.8.8", + json=self.response_get_asn_ripe, + ) + module_test.httpx_mock.add_response( + url="https://stat.ripe.net/data/whois/data.json?resource=15169", + json=self.response_get_asn_metadata_ripe, + ) + module_test.module.sources = ["ripe"] + + def check(self, module_test, events): + assert any(e.type == "ASN" for e in events) + assert any(e.type == "EMAIL_ADDRESS" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_1/module_tests/test_module_asset_inventory.py new file mode 100644 index 0000000000..b128fd6275 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_asset_inventory.py @@ -0,0 +1,17 @@ +from .base import ModuleTestBase + + +class TestAsset_Inventory(ModuleTestBase): + targets = ["8.8.8.8"] + config_overrides = {"dns_resolution": True} + + def check(self, module_test, events): + filename = next(module_test.scan.home.glob("asset-inventory.csv")) + with open(filename) as f: + assert "8.8.8.8,,8.8.8.8" in f.read() + filename = next(module_test.scan.home.glob("asset-inventory-ip-addresses-table*.txt")) + with open(filename) as f: + assert "8.8.0.0/16" in f.read() + filename = next(module_test.scan.home.glob("asset-inventory-domains-table*.txt")) + with open(filename) as f: + assert "dns.google" in f.read() diff --git a/bbot/test/test_step_1/module_tests/test_module_azure_tenant.py b/bbot/test/test_step_1/module_tests/test_module_azure_tenant.py new file mode 100644 index 0000000000..9e37b02069 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_azure_tenant.py @@ -0,0 +1,46 @@ +from .base import ModuleTestBase + + +class TestAzure_Tenant(ModuleTestBase): + tenant_response = """ + + + + http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetFederationInformationResponse + + 15 + 20 + 6411 + 14 + Exchange2015 + + + + + + NoError + + outlook.com + + blacklanternsecurity.onmicrosoft.com + + + + https://login.microsoftonline.com/extSTS.srf + urn:federation:MicrosoftOnline + + + + + +""" + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + method="POST", + url="https://autodiscover-s.outlook.com/autodiscover/autodiscover.svc", + text=self.tenant_response, + ) + + def check(self, module_test, events): + assert any(e.data == "blacklanternsecurity.onmicrosoft.com" and "affiliate" in e.tags for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_badsecrets.py b/bbot/test/test_step_1/module_tests/test_module_badsecrets.py new file mode 100644 index 0000000000..daa39f946a --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_badsecrets.py @@ -0,0 +1,110 @@ +from .base import ModuleTestBase + + +class TestBadSecrets(ModuleTestBase): + targets = [ + "http://127.0.0.1:8888/", + "http://127.0.0.1:8888/test.aspx", + "http://127.0.0.1:8888/cookie.aspx", + "http://127.0.0.1:8888/cookie2.aspx", + ] + + sample_viewstate = """ +
+
+ +
+ +
+ + + +
+
+ + +""" + + sample_viewstate_notvuln = """ +
+
+ +
+ +
+ + + +
+
+ + +""" + + modules_overrides = ["badsecrets", "httpx"] + + def setup_after_prep(self, module_test): + expect_args = {"uri": "/test.aspx"} + respond_args = {"response_data": self.sample_viewstate} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + respond_args = {"response_data": self.sample_viewstate_notvuln} + module_test.set_expect_requests(respond_args=respond_args) + + expect_args = {"uri": "/cookie.aspx"} + respond_args = { + "response_data": "

JWT Cookie Test

", + "headers": { + "set-cookie": "vulnjwt=eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo; secure" + }, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"uri": "/cookie2.aspx"} + respond_args = { + "response_data": "

Express Cookie Test

", + "headers": { + "set-cookie": "connect.sid=s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc; Path=/; Expires=Wed, 05 Apr 2023 04:47:29 GMT; HttpOnly" + }, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + SecretFound = False + IdentifyOnly = False + CookieBasedDetection = False + CookieBasedDetection_2 = False + + for e in events: + if ( + e.type == "VULNERABILITY" + and e.data["description"] + == "Known Secret Found. Secret Type: [ASP.NET MachineKey] Secret: [validationKey: 0F97BAE23F6F36801ABDB5F145124E00A6F795A97093D778EE5CD24F35B78B6FC4C0D0D4420657689C4F321F8596B59E83F02E296E970C4DEAD2DFE226294979 validationAlgo: SHA1 encryptionKey: 8CCFBC5B7589DD37DC3B4A885376D7480A69645DAEEC74F418B4877BEC008156 encryptionAlgo: AES] Product Type: [ASP.NET Viewstate] Product: [rJdyYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESf4nBu] Detecting Module: [ASPNET_Viewstate]" + ): + SecretFound = True + + if ( + e.type == "FINDING" + and e.data["description"] + == "Cryptographic Product identified. Product Type: [ASP.NET Viewstate] Product: [AAAAYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESfAAAA] Detecting Module: [ASPNET_Viewstate]" + ): + IdentifyOnly = True + + if ( + e.type == "VULNERABILITY" + and e.data["description"] + == "Known Secret Found. Secret Type: [HMAC/RSA Key] Secret: [1234] Product Type: [JSON Web Token (JWT)] Product: [eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo] Detecting Module: [Generic_JWT]" + ): + CookieBasedDetection = True + + if ( + e.type == "VULNERABILITY" + and e.data["description"] + == "Known Secret Found. Secret Type: [Express.js SESSION_SECRET] Secret: [keyboard cat] Product Type: [Express.js Signed Cookie] Product: [s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc] Detecting Module: [ExpressSignedCookies]" + ): + CookieBasedDetection_2 = True + + assert SecretFound, "No secret found" + assert IdentifyOnly, "No crypto product identified" + assert CookieBasedDetection, "No JWT cookie detected" + assert CookieBasedDetection_2, "No Express.js cookie detected" diff --git a/bbot/test/test_step_1/module_tests/test_module_bevigil.py b/bbot/test/test_step_1/module_tests/test_module_bevigil.py new file mode 100644 index 0000000000..bceeecc4a9 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bevigil.py @@ -0,0 +1,24 @@ +from .base import ModuleTestBase + + +class TestBeVigil(ModuleTestBase): + config_overrides = {"modules": {"bevigil": {"api_key": "asdf", "urls": True}}} + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/", + json={ + "domain": "blacklanternsecurity.com", + "subdomains": [ + "asdf.blacklanternsecurity.com", + ], + }, + ) + module_test.httpx_mock.add_response( + url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/urls/", + json={"domain": "blacklanternsecurity.com", "urls": ["https://asdf.blacklanternsecurity.com"]}, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "https://asdf.blacklanternsecurity.com/" for e in events), "Failed to detect url" diff --git a/bbot/test/test_step_1/module_tests/test_module_binaryedge.py b/bbot/test/test_step_1/module_tests/test_module_binaryedge.py new file mode 100644 index 0000000000..4845413da0 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_binaryedge.py @@ -0,0 +1,31 @@ +from .base import ModuleTestBase + + +class TestBinaryEdge(ModuleTestBase): + config_overrides = {"modules": {"binaryedge": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://api.binaryedge.io/v2/query/domains/subdomain/blacklanternsecurity.com", + json={ + "query": "blacklanternsecurity.com", + "page": 1, + "pagesize": 100, + "total": 1, + "events": [ + "asdf.blacklanternsecurity.com", + ], + }, + ) + module_test.httpx_mock.add_response( + url=f"https://api.binaryedge.io/v2/user/subscription", + json={ + "subscription": {"name": "Free"}, + "end_date": "2023-06-17", + "requests_left": 249, + "requests_plan": 250, + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_bucket_aws.py b/bbot/test/test_step_1/module_tests/test_module_bucket_aws.py new file mode 100644 index 0000000000..5be98dd86e --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bucket_aws.py @@ -0,0 +1,93 @@ +import re + +from .base import ModuleTestBase +from bbot.core.helpers.misc import rand_string + +__all__ = ["random_bucket_name_1", "random_bucket_name_2", "random_bucket_name_3", "Bucket_AWS_Base"] + +# first one is a normal bucket +random_bucket_name_1 = rand_string(15, digits=False) +# second one is open/vulnerable +random_bucket_name_2 = rand_string(15, digits=False) +# third one is a mutation +random_bucket_name_3 = f"{random_bucket_name_2}-dev" + + +class Bucket_AWS_Base(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + provider = "aws" + + random_bucket_1 = f"{random_bucket_name_1}.s3.amazonaws.com" + random_bucket_2 = f"{random_bucket_name_2}.s3-ap-southeast-2.amazonaws.com" + random_bucket_3 = f"{random_bucket_name_3}.s3.amazonaws.com" + + open_bucket_body = """ + vpn-static1000falsestyle.css2017-03-18T06:41:59.000Z"bf9e72bdab09b785f05ff0395023cc35"429STANDARD""" + + @property + def config_overrides(self): + return {"modules": {f"bucket_{self.provider}": {"permutations": True}}} + + @property + def modules_overrides(self): + return ["excavate", "speculate", "httpx", f"bucket_{self.provider}"] + + def url_setup(self): + self.url_1 = f"https://{self.random_bucket_1}" + self.url_2 = f"https://{self.random_bucket_2}" + self.url_3 = f"https://{self.random_bucket_3}" + + def bucket_setup(self): + self.url_setup() + self.website_body = f""" + + + """ + + def setup_after_prep(self, module_test): + self.bucket_setup() + # patch mutations + module_test.scan.helpers.word_cloud.mutations = lambda b, cloud=False: [ + (b, "dev"), + ] + module_test.set_expect_requests( + expect_args={"method": "GET", "uri": "/"}, respond_args={"response_data": self.website_body} + ) + if module_test.module.supports_open_check: + module_test.httpx_mock.add_response( + url=self.url_2, + text=self.open_bucket_body, + ) + module_test.httpx_mock.add_response( + url=self.url_3, + text="", + ) + module_test.httpx_mock.add_response(url=re.compile(".*"), text="", status_code=404) + + def check(self, module_test, events): + # make sure buckets were excavated + assert any( + e.type == "STORAGE_BUCKET" and str(e.module) == f"{self.provider}_cloud" for e in events + ), f'bucket not found for provider "{self.provider}"' + # make sure open buckets were found + if module_test.module.supports_open_check: + assert any( + e.type == "FINDING" and str(e.module) == f"bucket_{self.provider}" for e in events + ), f'open bucket not found for provider "{self.provider}"' + for e in events: + if e.type == "FINDING" and str(e.module) == f"bucket_{self.provider}": + url = e.data.get("url", "") + assert self.random_bucket_2 in url + assert not self.random_bucket_1 in url + assert not f"{self.random_bucket_3}" in url + # make sure bucket mutations were found + assert any( + e.type == "STORAGE_BUCKET" + and str(e.module) == f"bucket_{self.provider}" + and f"{random_bucket_name_3}" in e.data["url"] + for e in events + ), f'bucket (dev mutation) not found for provider "{self.provider}"' + + +class TestBucket_AWS(Bucket_AWS_Base): + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_bucket_azure.py b/bbot/test/test_step_1/module_tests/test_module_bucket_azure.py new file mode 100644 index 0000000000..3081462ba8 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bucket_azure.py @@ -0,0 +1,13 @@ +from .test_module_bucket_aws import * + + +class TestBucket_Azure(Bucket_AWS_Base): + provider = "azure" + random_bucket_1 = f"{random_bucket_name_1}.blob.core.windows.net" + random_bucket_2 = f"{random_bucket_name_2}.blob.core.windows.net" + random_bucket_3 = f"{random_bucket_name_3}.blob.core.windows.net" + + def url_setup(self): + self.url_1 = f"https://{self.random_bucket_1}" + self.url_2 = f"https://{self.random_bucket_2}" + self.url_3 = f"https://{self.random_bucket_3}/{random_bucket_name_3}?restype=container" diff --git a/bbot/test/test_step_1/module_tests/test_module_bucket_digitalocean.py b/bbot/test/test_step_1/module_tests/test_module_bucket_digitalocean.py new file mode 100644 index 0000000000..18d98b6413 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bucket_digitalocean.py @@ -0,0 +1,10 @@ +from .test_module_bucket_aws import * + + +class TestBucket_DigitalOcean(Bucket_AWS_Base): + provider = "digitalocean" + random_bucket_1 = f"{random_bucket_name_1}.fra1.digitaloceanspaces.com" + random_bucket_2 = f"{random_bucket_name_2}.fra1.digitaloceanspaces.com" + random_bucket_3 = f"{random_bucket_name_3}.fra1.digitaloceanspaces.com" + + open_bucket_body = """cloud011000falsetest.doc2020-10-14T15:23:37.545Z"4d25c8699f7347acc9f41e57148c62c0"13362425STANDARD19578831957883Normal""" diff --git a/bbot/test/test_step_1/module_tests/test_module_bucket_firebase.py b/bbot/test/test_step_1/module_tests/test_module_bucket_firebase.py new file mode 100644 index 0000000000..f78f6cbc58 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bucket_firebase.py @@ -0,0 +1,13 @@ +from .test_module_bucket_aws import * + + +class TestBucket_Firebase(Bucket_AWS_Base): + provider = "firebase" + random_bucket_1 = f"{random_bucket_name_1}.firebaseio.com" + random_bucket_2 = f"{random_bucket_name_2}.firebaseio.com" + random_bucket_3 = f"{random_bucket_name_3}.firebaseio.com" + + def url_setup(self): + self.url_1 = f"https://{self.random_bucket_1}" + self.url_2 = f"https://{self.random_bucket_2}/.json" + self.url_3 = f"https://{self.random_bucket_3}/.json" diff --git a/bbot/test/test_step_1/module_tests/test_module_bucket_gcp.py b/bbot/test/test_step_1/module_tests/test_module_bucket_gcp.py new file mode 100644 index 0000000000..49e35b52e2 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bucket_gcp.py @@ -0,0 +1,27 @@ +from .test_module_bucket_aws import * + + +class TestBucket_GCP(Bucket_AWS_Base): + provider = "gcp" + random_bucket_1 = f"{random_bucket_name_1}.storage.googleapis.com" + random_bucket_2 = f"{random_bucket_name_2}.storage.googleapis.com" + random_bucket_3 = f"{random_bucket_name_3}.storage.googleapis.com" + open_bucket_body = """{ + "kind": "storage#testIamPermissionsResponse", + "permissions": [ + "storage.objects.create", + "storage.objects.list" + ] +}""" + + def bucket_setup(self): + self.url_setup() + self.website_body = f""" + + + """ + + def url_setup(self): + self.url_1 = f"{random_bucket_name_1}.storage.googleapis.com" + self.url_2 = f"https://www.googleapis.com/storage/v1/b/{random_bucket_name_2}/iam/testPermissions?permissions=storage.buckets.setIamPolicy&permissions=storage.objects.list&permissions=storage.objects.get&permissions=storage.objects.create" + self.url_3 = f"https://www.googleapis.com/storage/v1/b/{random_bucket_name_3}" diff --git a/bbot/test/test_step_1/module_tests/test_module_builtwith.py b/bbot/test/test_step_1/module_tests/test_module_builtwith.py new file mode 100644 index 0000000000..5ff69993fd --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_builtwith.py @@ -0,0 +1,110 @@ +from .base import ModuleTestBase + + +class TestBuiltWith(ModuleTestBase): + config_overrides = {"modules": {"builtwith": {"api_key": "asdf"}}} + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://api.builtwith.com/v20/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes", + json={ + "Results": [ + { + "Result": { + "IsDB": "True", + "Spend": 734, + "Paths": [ + { + "Technologies": [ + { + "Name": "nginx", + "Tag": "Web Server", + "FirstDetected": 1533510000000, + "LastDetected": 1559516400000, + "IsPremium": "no", + }, + { + "Parent": "nginx", + "Name": "Nginx 1.14", + "Tag": "Web Server", + "FirstDetected": 1555542000000, + "LastDetected": 1559516400000, + "IsPremium": "no", + }, + { + "Name": "Domain Not Resolving", + "Tag": "hosting", + "FirstDetected": 1613894400000, + "LastDetected": 1633244400000, + "IsPremium": "no", + }, + ], + "FirstIndexed": 1533510000000, + "LastIndexed": 1633244400000, + "Domain": "blacklanternsecurity.com", + "Url": "", + "SubDomain": "asdf", + } + ], + }, + "Meta": { + "Majestic": 0, + "Umbrella": 0, + "Vertical": "", + "Social": None, + "CompanyName": None, + "Telephones": None, + "Emails": [], + "City": None, + "State": None, + "Postcode": None, + "Country": "US", + "Names": None, + "ARank": 6249242, + "QRank": -1, + }, + "Attributes": { + "Employees": 0, + "MJRank": 0, + "MJTLDRank": 0, + "RefSN": 0, + "RefIP": 0, + "Followers": 0, + "Sitemap": 0, + "GTMTags": 0, + "QubitTags": 0, + "TealiumTags": 0, + "AdobeTags": 0, + "CDimensions": 0, + "CGoals": 0, + "CMetrics": 0, + "ProductCount": 0, + }, + "FirstIndexed": 1389481200000, + "LastIndexed": 1684220400000, + "Lookup": "blacklanternsecurity.com", + "SalesRevenue": 0, + } + ], + "Errors": [], + "Trust": None, + }, + ) + module_test.httpx_mock.add_response( + url=f"https://api.builtwith.com/redirect1/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com", + json={ + "Lookup": "blacklanternsecurity.com", + "Inbound": [ + { + "Domain": "blacklanternsecurity.github.io", + "FirstDetected": 1564354800000, + "LastDetected": 1683783431121, + } + ], + "Outbound": None, + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "blacklanternsecurity.github.io" for e in events), "Failed to detect redirect" diff --git a/bbot/test/test_step_1/module_tests/test_module_bypass403.py b/bbot/test/test_step_1/module_tests/test_module_bypass403.py new file mode 100644 index 0000000000..7446352319 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_bypass403.py @@ -0,0 +1,50 @@ +import re +from .base import ModuleTestBase + + +class TestBypass403(ModuleTestBase): + targets = ["http://127.0.0.1:8888/test"] + modules_overrides = ["bypass403", "httpx"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/test..;/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + module_test.httpserver.no_handler_status_code = 403 + + def check(self, module_test, events): + findings = [e for e in events if e.type == "FINDING"] + assert len(findings) == 1 + finding = findings[0] + assert "http://127.0.0.1:8888/test..;/" in finding.data["description"] + + +class TestBypass403_aspnetcookieless(ModuleTestBase): + targets = ["http://127.0.0.1:8888/admin.aspx"] + modules_overrides = ["bypass403", "httpx"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": re.compile(r"\/\([sS]\(\w+\)\)\/.+\.aspx")} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + module_test.httpserver.no_handler_status_code = 403 + + def check(self, module_test, events): + findings = [e for e in events if e.type == "FINDING"] + assert len(findings) == 2 + assert all("(S(X))/admin.aspx" in e.data["description"] for e in findings) + + +class TestBypass403_waf(ModuleTestBase): + targets = ["http://127.0.0.1:8888/test"] + modules_overrides = ["bypass403", "httpx"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/test..;/"} + respond_args = {"response_data": "The requested URL was rejected"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + module_test.httpserver.no_handler_status_code = 403 + + def check(self, module_test, events): + findings = [e for e in events if e.type == "FINDING"] + assert not any(findings) diff --git a/bbot/test/test_step_1/module_tests/test_module_c99.py b/bbot/test/test_step_1/module_tests/test_module_c99.py new file mode 100644 index 0000000000..98c5e87652 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_c99.py @@ -0,0 +1,25 @@ +from .base import ModuleTestBase + + +class TestC99(ModuleTestBase): + config_overrides = {"modules": {"c99": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.c99.nl/randomnumber?key=asdf&between=1,100&json", + json={"success": True, "output": 65}, + ) + module_test.httpx_mock.add_response( + url="https://api.c99.nl/subdomainfinder?key=asdf&domain=blacklanternsecurity.com&json", + json={ + "success": True, + "subdomains": [ + {"subdomain": "asdf.blacklanternsecurity.com", "ip": "1.2.3.4", "cloudflare": True}, + ], + "cached": True, + "cache_time": "2023-05-19 03:13:05", + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_censys.py b/bbot/test/test_step_1/module_tests/test_module_censys.py new file mode 100644 index 0000000000..e2792e6131 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_censys.py @@ -0,0 +1,41 @@ +from .base import ModuleTestBase + + +class TestCensys(ModuleTestBase): + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://search.censys.io/api/v1/account", + json={ + "email": "info@blacklanternsecurity.com", + "login": "nope", + "first_login": "1917-08-03 20:03:55", + "last_login": "1918-05-19 01:15:22", + "quota": {"used": 26, "allowance": 250, "resets_at": "1919-06-03 16:30:32"}, + }, + ) + module_test.httpx_mock.add_response( + url="https://search.censys.io/api/v1/search/certificates", + match_content=b'{"query": "parsed.names: blacklanternsecurity.com", "page": 1, "fields": ["parsed.names"]}', + json={ + "status": "ok", + "metadata": { + "query": "parsed.names: blacklanternsecurity.com", + "count": 1, + "backend_time": 4465, + "page": 1, + "pages": 4, + }, + "results": [ + { + "parsed.names": [ + "asdf.blacklanternsecurity.com", + "zzzz.blacklanternsecurity.com", + ] + }, + ], + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_certspotter.py b/bbot/test/test_step_1/module_tests/test_module_certspotter.py new file mode 100644 index 0000000000..fb6bb002c6 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_certspotter.py @@ -0,0 +1,14 @@ +from .base import ModuleTestBase + + +class TestCertspotter(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.module.abort_if = lambda e: False + for t in self.targets: + module_test.httpx_mock.add_response( + url="https://api.certspotter.com/v1/issuances?domain=blacklanternsecurity.com&include_subdomains=true&expand=dns_names", + json=[{"dns_names": ["*.asdf.blacklanternsecurity.com"]}], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_crobat.py b/bbot/test/test_step_1/module_tests/test_module_crobat.py new file mode 100644 index 0000000000..c797541c27 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_crobat.py @@ -0,0 +1,12 @@ +from .base import ModuleTestBase + + +class TestCrobat(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://sonar.omnisint.io/subdomains/blacklanternsecurity.com", + json=["asdf.blacklanternsecurity.com"], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_crt.py b/bbot/test/test_step_1/module_tests/test_module_crt.py new file mode 100644 index 0000000000..f7a13e98de --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_crt.py @@ -0,0 +1,15 @@ +from .base import ModuleTestBase + + +class TestCRT(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.module.abort_if = lambda e: False + for t in self.targets: + module_test.httpx_mock.add_response( + url="https://crt.sh?q=%25.blacklanternsecurity.com&output=json", + json=[{"id": 1, "name_value": "asdf.blacklanternsecurity.com\nzzzz.blacklanternsecurity.com"}], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_csv.py b/bbot/test/test_step_1/module_tests/test_module_csv.py new file mode 100644 index 0000000000..fc180d481b --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_csv.py @@ -0,0 +1,8 @@ +from .base import ModuleTestBase + + +class TestCSV(ModuleTestBase): + def check(self, module_test, events): + csv_file = module_test.scan.home / "output.csv" + with open(csv_file) as f: + assert "DNS_NAME,blacklanternsecurity.com,,TARGET" in f.read() diff --git a/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py new file mode 100644 index 0000000000..4b2617a6bb --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py @@ -0,0 +1,19 @@ +from .base import ModuleTestBase + + +class TestDNSCommonSRV(ModuleTestBase): + def setup_after_prep(self, module_test): + old_resolve_fn = module_test.scan.helpers.dns.resolve + + async def resolve(query, **kwargs): + if query == "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" and kwargs.get("type", "").upper() == "SRV": + return {"asdf.blacklanternsecurity.com"} + return await old_resolve_fn(query, **kwargs) + + module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "resolve", resolve) + + def check(self, module_test, events): + assert any( + e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" for e in events + ), "Failed to detect subdomain" + assert not any(e.data == "_ldap._tcp.dc._msdcs.blacklanternsecurity.com" for e in events), "False positive" diff --git a/bbot/test/test_step_1/module_tests/test_module_dnsdumpster.py b/bbot/test/test_step_1/module_tests/test_module_dnsdumpster.py new file mode 100644 index 0000000000..b7b1067cf4 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_dnsdumpster.py @@ -0,0 +1,18 @@ +from .base import ModuleTestBase + + +class TestDNSDumpster(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://dnsdumpster.com", + headers={"Set-Cookie": "csrftoken=asdf"}, + content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n
\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similiar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n
\n\n\n
\n
\n
\n
\n

About

\n

At hackertarget.com, we have been scanning the Internet since 2007. Our vulnerability scanners now reach millions of IP addresses every year. We work to raise awareness of the value in open source security solutions.

\n
\n
\n
\n
\n

Stay in Touch

\n\n
\n

dnsdumpster@gmail.com

\n
\n
\n

@hackertarget

\n
\n\n\n\n
\n
\n
\n\n \n \n
\n
Low volume Updates and News
\n
\n
\n
\n\n \n\n
\n
\n
\n
\n
\n

Copyright © 2023 Hacker Target Pty Ltd

\n
\n
\n\n\n
\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n\n\n\n\n\n\n \n\n', + ) + module_test.httpx_mock.add_response( + url=f"https://dnsdumpster.com/", + method="POST", + content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n
\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n

Showing results for blacklanternsecurity.com

\n
\n
\n
\n
\n

Hosting (IP block owners)

\n
\n
\n

GeoIP of Host Locations

\n
\n
\n
\n\n

DNS Servers

\n
\n \n \n \n \n \n \n
ns01.domaincontrol.com.
\n\n\n \n
\n
\n		97.74.100.1
ns01.domaincontrol.com		GODADDY-DNS
United States
ns02.domaincontrol.com.
\n\n\n \n
\n
\n		173.201.68.1
ns02.domaincontrol.com		GODADDY-DNS
United States
\n
\n\n

MX Records ** This is where email for the domain goes...

\n
\n \n \n \n \n
asdf.blacklanternsecurity.com.mail.protection.outlook.com.
\n\n\n
\n
\n		104.47.55.138
mail-bn8nam120138.inbound.protection.outlook.com		MICROSOFT-CORP-MSN-AS-BLOCK
United States
\n
\n\n

TXT Records ** Find more hosts in Sender Policy Framework (SPF) configurations

\n
\n \n\n\n\n\n\n\n\n\n\n
"MS=ms26206678"
"v=spf1 ip4:50.240.76.25 include:spf.protection.outlook.com -all"
"google-site-verification=O_PoQFTGJ_hZ9LqfNT9OEc0KPFERKHQ_1t1m0YTx_1E"
"google-site-verification=7XKUMxJSTHBSzdvT7gH47jLRjNAS76nrEfXmzhR_DO4"
\n
\n\n\n

Host Records (A) ** this data may not be current as it uses a static database (updated monthly)

\n
\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n GitHub.com\n\n\n\n\n\n\n\n\n
HTTP TECH: \n varnish\n\n\n\n		185.199.108.153
cdn-185-199-108-153.github.com		FASTLY
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n
SSH: \n SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.3\n\n\n\n\n\n\n\n		143.244.156.80
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n Apache/2.4.29 (Ubuntu)\n\n\n\n\n\n\n\n\n
HTTP TECH: \n Ubuntu
Apache,2.4.29\n\n\n\n		64.227.8.231
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		192.34.56.157
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		192.241.216.208
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		167.71.95.71
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n		157.245.247.197
asdf.blacklanternsecurity.com		DIGITALOCEAN-ASN
United States
\n
\n\n\n\n\n\n
\n

Mapping the domain ** click for full size image

\n

\n\n

\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similiar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n\n\n\n\n
\n\n\n
\n
\n
\n
\n

About

\n

At hackertarget.com, we have been scanning the Internet since 2007. Our vulnerability scanners now reach millions of IP addresses every year. We work to raise awareness of the value in open source security solutions.

\n
\n
\n
\n
\n

Stay in Touch

\n\n
\n

dnsdumpster@gmail.com

\n
\n
\n

@hackertarget

\n
\n\n\n\n
\n
\n
\n\n \n \n
\n
Low volume Updates and News
\n
\n
\n
\n\n \n\n
\n
\n
\n
\n
\n

Copyright © 2023 Hacker Target Pty Ltd

\n
\n
\n\n\n
\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n \n \n \n\n \n\n\n\n\n\n\n\n\n\n\n\n\n \n\n', + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py new file mode 100644 index 0000000000..f746ee717d --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py @@ -0,0 +1,56 @@ +import dns.zone +import dns.query +import dns.message +from types import SimpleNamespace + +from .base import ModuleTestBase + + +class TestDNSZoneTransfer(ModuleTestBase): + targets = ["blacklanternsecurity.fakedomain"] + config_overrides = {"dns_resolution": True} + + def setup_after_prep(self, module_test): + old_resolve_fn = module_test.scan.helpers.dns._resolve_hostname + + class MockRecord: + def __init__(self, record, rdtype): + self.rdtype = SimpleNamespace() + self.rdtype.name = rdtype + self.record = record + + def __str__(self): + return self.record + + def to_text(self): + return str(self) + + async def _resolve_hostname(query, **kwargs): + if query == "blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() == "NS": + return [MockRecord("ns01.blacklanternsecurity.fakedomain", "NS")], [] + if query == "ns01.blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() in "A": + return [MockRecord("127.0.0.1", "A")], [] + return await old_resolve_fn(query, **kwargs) + + def from_xfr(*args, **kwargs): + zone_text = """ +@ 600 IN SOA ns.blacklanternsecurity.fakedomain. admin.blacklanternsecurity.fakedomain. ( + 1 ; Serial + 3600 ; Refresh + 900 ; Retry + 604800 ; Expire + 86400 ) ; Minimum TTL +@ 600 IN NS ns.blacklanternsecurity.fakedomain. +@ 600 IN A 127.0.0.1 +asdf 600 IN A 127.0.0.1 +zzzz 600 IN AAAA dead::beef +""" + zone = dns.zone.from_text(zone_text, origin="blacklanternsecurity.fakedomain.") + return zone + + module_test.monkeypatch.setattr("dns.zone.from_xfr", from_xfr) + module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "_resolve_hostname", _resolve_hostname) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.fakedomain" for e in events), "Zone transfer failed" + assert any(e.data == "zzzz.blacklanternsecurity.fakedomain" for e in events), "Zone transfer failed" diff --git a/bbot/test/test_step_1/module_tests/test_module_emailformat.py b/bbot/test/test_step_1/module_tests/test_module_emailformat.py new file mode 100644 index 0000000000..ff1c0d3b36 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_emailformat.py @@ -0,0 +1,12 @@ +from .base import ModuleTestBase + + +class TestEmailFormat(ModuleTestBase): + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://www.email-format.com/d/blacklanternsecurity.com/", + text="

info@blacklanternsecurity.com", + ) + + def check(self, module_test, events): + assert any(e.data == "info@blacklanternsecurity.com" for e in events), "Failed to detect email" diff --git a/bbot/test/test_step_1/module_tests/test_module_excavate.py b/bbot/test/test_step_1/module_tests/test_module_excavate.py new file mode 100644 index 0000000000..250506f5e1 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_excavate.py @@ -0,0 +1,141 @@ +from .base import ModuleTestBase + + +class TestExcavate(ModuleTestBase): + targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/links.html"] + modules_overrides = ["excavate", "httpx"] + config_overrides = {"web_spider_distance": 1, "web_spider_depth": 1} + + def setup_before_prep(self, module_test): + response_data = """ + ftp://ftp.test.notreal + \\nhttps://www1.test.notreal + \\x3dhttps://www2.test.notreal + %0ahttps://www3.test.notreal + \\u000ahttps://www4.test.notreal + \nwww5.test.notreal + \\x3dwww6.test.notreal + %0awww7.test.notreal + \\u000awww8.test.notreal + + """ + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": response_data} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # verify relatives path a-tag parsing is working correctly + + expect_args = {"method": "GET", "uri": "/subdir/links.html"} + respond_args = {"response_data": ""} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/relative.html"} + respond_args = {"response_data": ""} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + module_test.httpserver.no_handler_status_code = 404 + + def check(self, module_test, events): + event_data = [e.data for e in events] + assert "https://www1.test.notreal/" in event_data + assert "https://www2.test.notreal/" in event_data + assert "https://www3.test.notreal/" in event_data + assert "https://www4.test.notreal/" in event_data + assert "www1.test.notreal" in event_data + assert "www2.test.notreal" in event_data + assert "www3.test.notreal" in event_data + assert "www4.test.notreal" in event_data + assert "www5.test.notreal" in event_data + assert "www6.test.notreal" in event_data + assert "www7.test.notreal" in event_data + assert "www8.test.notreal" in event_data + assert "http://www9.test.notreal/" in event_data + + assert "nhttps://www1.test.notreal/" not in event_data + assert "x3dhttps://www2.test.notreal/" not in event_data + assert "a2https://www3.test.notreal/" not in event_data + assert "uac20https://www4.test.notreal/" not in event_data + assert "nwww5.test.notreal" not in event_data + assert "x3dwww6.test.notreal" not in event_data + assert "a2www7.test.notreal" not in event_data + assert "uac20www8.test.notreal" not in event_data + + assert any( + e.type == "FINDING" and e.data.get("description", "") == "Non-HTTP URI: ftp://ftp.test.notreal" + for e in events + ) + assert any( + e.type == "PROTOCOL" + and e.data.get("protocol", "") == "FTP" + and e.data.get("host", "") == "ftp.test.notreal" + for e in events + ) + + assert any( + e.type == "URL_UNVERIFIED" + and e.data == "http://127.0.0.1:8888/relative.html" + and "spider-danger" not in e.tags + for e in events + ) + + assert any( + e.type == "URL_UNVERIFIED" + and e.data == "http://127.0.0.1:8888/2/depth2.html" + and "spider-danger" in e.tags + for e in events + ) + + assert any( + e.type == "URL_UNVERIFIED" + and e.data == "http://127.0.0.1:8888/distance2.html" + and "spider-danger" in e.tags + for e in events + ) + + +class TestExcavate2(TestExcavate): + targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/"] + + def setup_before_prep(self, module_test): + # root relative + expect_args = {"method": "GET", "uri": "/rootrelative.html"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # page relative + expect_args = {"method": "GET", "uri": "/subdir/pagerelative.html"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/subdir/"} + respond_args = { + "response_data": "root relativepage relative" + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + module_test.httpserver.no_handler_status_code = 404 + + def check(self, module_test, events): + root_relative_detection = False + page_relative_detection = False + root_page_confusion_1 = False + root_page_confusion_2 = False + + for e in events: + if e.type == "URL_UNVERIFIED": + # these cases represent the desired behavior for parsing relative links + if e.data == "http://127.0.0.1:8888/rootrelative.html": + root_relative_detection = True + if e.data == "http://127.0.0.1:8888/subdir/pagerelative.html": + page_relative_detection = True + + # these cases indicates that excavate parsed the relative links incorrectly + if e.data == "http://127.0.0.1:8888/pagerelative.html": + root_page_confusion_1 = True + if e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": + root_page_confusion_2 = True + + assert root_relative_detection, "Failed to properly excavate root-relative URL" + assert page_relative_detection, "Failed to properly excavate page-relative URL" + assert not root_page_confusion_1, "Incorrectly detected page-relative URL" + assert not root_page_confusion_2, "Incorrectly detected root-relative URL" diff --git a/bbot/test/test_step_1/module_tests/test_module_ffuf.py b/bbot/test/test_step_1/module_tests/test_module_ffuf.py new file mode 100644 index 0000000000..9da9805567 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_ffuf.py @@ -0,0 +1,45 @@ +from .base import ModuleTestBase, tempwordlist + + +class TestFFUF(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + test_wordlist = ["11111111", "admin", "junkword1", "zzzjunkword2"] + config_overrides = { + "modules": { + "ffuf": { + "wordlist": tempwordlist(test_wordlist), + } + } + } + modules_overrides = ["ffuf", "httpx"] + + def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/admin"} + respond_args = {"response_data": "alive admin page"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "URL_UNVERIFIED" and "admin" in e.data for e in events) + assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) + + +class TestFFUF2(TestFFUF): + test_wordlist = ["11111111", "console", "junkword1", "zzzjunkword2"] + config_overrides = {"modules": {"ffuf": {"wordlist": tempwordlist(test_wordlist), "extensions": "php"}}} + + def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/console.php"} + respond_args = {"response_data": "alive admin page"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "URL_UNVERIFIED" and "console" in e.data for e in events) + assert not any(e.type == "URL_UNVERIFIED" and "11111111" in e.data for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_ffuf_shortnames.py b/bbot/test/test_step_1/module_tests/test_module_ffuf_shortnames.py new file mode 100644 index 0000000000..09312dcee3 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_ffuf_shortnames.py @@ -0,0 +1,208 @@ +from .base import ModuleTestBase, tempwordlist + + +class TestFFUFShortnames(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + test_wordlist = ["11111111", "administrator", "portal", "console", "junkword1", "zzzjunkword2", "directory"] + config_overrides = { + "modules": { + "ffuf_shortnames": { + "find_common_prefixes": True, + "find_common_prefixes": True, + "wordlist": tempwordlist(test_wordlist), + } + } + } + modules_overrides = ["ffuf_shortnames", "httpx"] + + def setup_after_prep(self, module_test): + module_test.httpserver.no_handler_status_code = 404 + + seed_events = [] + parent_event = module_test.scan.make_event( + "http://127.0.0.1:8888/", + "URL", + module_test.scan.root_event, + module="httpx", + tags=["status-200", "distance-0"], + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ADMINI~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ADM_PO~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ABCZZZ~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ABCXXX~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ABCYYY~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ABCCON~1.ASP", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/DIRECT~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/ADM_DI~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/XYZDIR~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/XYZAAA~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/XYZBBB~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/XYZCCC~1", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-directory"], + ) + ) + seed_events.append( + module_test.scan.make_event( + "http://127.0.0.1:8888/SHORT~1.PL", + "URL_HINT", + parent_event, + module="iis_shortnames", + tags=["shortname-file"], + ) + ) + module_test.scan.target._events["http://127.0.0.1:8888"] = seed_events + + expect_args = {"method": "GET", "uri": "/administrator.aspx"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/adm_portal.aspx"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/abcconsole.aspx"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/directory/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/adm_directory/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/xyzdirectory/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/short.pl"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + basic_detection = False + directory_detection = False + prefix_detection = False + delimeter_detection = False + directory_delimeter_detection = False + prefix_delimeter_detection = False + short_extensions_detection = False + + for e in events: + if e.type == "URL_UNVERIFIED": + if e.data == "http://127.0.0.1:8888/administrator.aspx": + basic_detection = True + if e.data == "http://127.0.0.1:8888/directory/": + directory_detection = True + if e.data == "http://127.0.0.1:8888/adm_portal.aspx": + prefix_detection = True + if e.data == "http://127.0.0.1:8888/abcconsole.aspx": + delimeter_detection = True + if e.data == "http://127.0.0.1:8888/abcconsole.aspx": + directory_delimeter_detection = True + if e.data == "http://127.0.0.1:8888/xyzdirectory/": + prefix_delimeter_detection = True + if e.data == "http://127.0.0.1:8888/short.pl": + short_extensions_detection = True + + assert basic_detection + assert directory_detection + assert prefix_detection + assert delimeter_detection + assert directory_delimeter_detection + assert prefix_delimeter_detection + assert short_extensions_detection diff --git a/bbot/test/test_step_1/module_tests/test_module_fingerprintx.py b/bbot/test/test_step_1/module_tests/test_module_fingerprintx.py new file mode 100644 index 0000000000..7e0cc3a169 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_fingerprintx.py @@ -0,0 +1,14 @@ +from .base import ModuleTestBase + + +class TestFingerprintx(ModuleTestBase): + targets = ["127.0.0.1:8888"] + + def check(self, module_test, events): + assert any( + event.type == "PROTOCOL" + and event.host == module_test.scan.helpers.make_ip_type("127.0.0.1") + and event.port == 8888 + and event.data["protocol"] == "HTTP" + for event in events + ), "HTTP protocol not detected" diff --git a/bbot/test/test_step_1/module_tests/test_module_fullhunt.py b/bbot/test/test_step_1/module_tests/test_module_fullhunt.py new file mode 100644 index 0000000000..bada14732a --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_fullhunt.py @@ -0,0 +1,48 @@ +from .base import ModuleTestBase + + +class TestFullhunt(ModuleTestBase): + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://fullhunt.io/api/v1/auth/status", + json={ + "message": "", + "status": 200, + "user": { + "company": "nightwatch", + "email": "jonsnow@nightwatch.notreal", + "first_name": "Jon", + "last_name": "Snow", + "plan": "free", + }, + "user_credits": { + "credits_usage": 0, + "max_results_per_request": 3000, + "remaining_credits": 100, + "total_credits_per_month": 100, + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://fullhunt.io/api/v1/domain/blacklanternsecurity.com/subdomains", + json={ + "domain": "blacklanternsecurity.com", + "hosts": [ + "asdf.blacklanternsecurity.com", + ], + "message": "", + "metadata": { + "all_results_count": 11, + "available_results_for_user": 11, + "domain": "blacklanternsecurity.com", + "last_scanned": 1647083421, + "max_results_for_user": 3000, + "timestamp": 1684541940, + "user_plan": "free", + }, + "status": 200, + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_github.py b/bbot/test/test_step_1/module_tests/test_module_github.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_gowitness.py b/bbot/test/test_step_1/module_tests/test_module_gowitness.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_hackertarget.py b/bbot/test/test_step_1/module_tests/test_module_hackertarget.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_host_header.py b/bbot/test/test_step_1/module_tests/test_module_host_header.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_http.py b/bbot/test/test_step_1/module_tests/test_module_http.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_httpx.py b/bbot/test/test_step_1/module_tests/test_module_httpx.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_human.py b/bbot/test/test_step_1/module_tests/test_module_human.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_hunt.py b/bbot/test/test_step_1/module_tests/test_module_hunt.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_hunterio.py b/bbot/test/test_step_1/module_tests/test_module_hunterio.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py b/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py b/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_ipstack.py b/bbot/test/test_step_1/module_tests/test_module_ipstack.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_json.py b/bbot/test/test_step_1/module_tests/test_module_json.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_leakix.py b/bbot/test/test_step_1/module_tests/test_module_leakix.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_masscan.py b/bbot/test/test_step_1/module_tests/test_module_masscan.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_massdns.py b/bbot/test/test_step_1/module_tests/test_module_massdns.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_naabu.py b/bbot/test/test_step_1/module_tests/test_module_naabu.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_neo4j.py b/bbot/test/test_step_1/module_tests/test_module_neo4j.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_ntlm.py b/bbot/test/test_step_1/module_tests/test_module_ntlm.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_nuclei.py b/bbot/test/test_step_1/module_tests/test_module_nuclei.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_otx.py b/bbot/test/test_step_1/module_tests/test_module_otx.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_passivetotal.py b/bbot/test/test_step_1/module_tests/test_module_passivetotal.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_pgp.py b/bbot/test/test_step_1/module_tests/test_module_pgp.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_python.py b/bbot/test/test_step_1/module_tests/test_module_python.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_rapiddns.py b/bbot/test/test_step_1/module_tests/test_module_rapiddns.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_riddler.py b/bbot/test/test_step_1/module_tests/test_module_riddler.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_robots.py b/bbot/test/test_step_1/module_tests/test_module_robots.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_secretsdb.py b/bbot/test/test_step_1/module_tests/test_module_secretsdb.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_securitytrails.py b/bbot/test/test_step_1/module_tests/test_module_securitytrails.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_services.py b/bbot/test/test_step_1/module_tests/test_module_services.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py b/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_skymem.py b/bbot/test/test_step_1/module_tests/test_module_skymem.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_smuggler.py b/bbot/test/test_step_1/module_tests/test_module_smuggler.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_social.py b/bbot/test/test_step_1/module_tests/test_module_social.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_speculate.py b/bbot/test/test_step_1/module_tests/test_module_speculate.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_sslcert.py b/bbot/test/test_step_1/module_tests/test_module_sslcert.py new file mode 100644 index 0000000000..d443dfb618 --- /dev/null +++ b/bbot/test/test_step_1/module_tests/test_module_sslcert.py @@ -0,0 +1,8 @@ +from .base import ModuleTestBase + + +class TestSSLCert(ModuleTestBase): + targets = ["8.8.8.8:443"] + + def check(self, module_test, events): + assert any(e.data == "dns.google" and str(e.module) == "sslcert" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py b/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_sublist3r.py b/bbot/test/test_step_1/module_tests/test_module_sublist3r.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_telerik.py b/bbot/test/test_step_1/module_tests/test_module_telerik.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_threatminer.py b/bbot/test/test_step_1/module_tests/test_module_threatminer.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py b/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_urlscan.py b/bbot/test/test_step_1/module_tests/test_module_urlscan.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_vhost.py b/bbot/test/test_step_1/module_tests/test_module_vhost.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_viewdns.py b/bbot/test/test_step_1/module_tests/test_module_viewdns.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_virustotal.py b/bbot/test/test_step_1/module_tests/test_module_virustotal.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_wafw00f.py b/bbot/test/test_step_1/module_tests/test_module_wafw00f.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py b/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_wayback.py b/bbot/test/test_step_1/module_tests/test_module_wayback.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_web_report.py b/bbot/test/test_step_1/module_tests/test_module_web_report.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_websocket.py b/bbot/test/test_step_1/module_tests/test_module_websocket.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bbot/test/test_step_1/module_tests/test_module_zoomeye.py b/bbot/test/test_step_1/module_tests/test_module_zoomeye.py new file mode 100644 index 0000000000..e69de29bb2 From cb95e10efc2c0f84427f0392bec3988eb77b647b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 May 2023 20:25:49 -0400 Subject: [PATCH 029/387] removed duplicate tests --- bbot/test/test_step_1/module_tests/__init__.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/bbot/test/test_step_1/module_tests/__init__.py b/bbot/test/test_step_1/module_tests/__init__.py index 0b098d46a2..e69de29bb2 100644 --- a/bbot/test/test_step_1/module_tests/__init__.py +++ b/bbot/test/test_step_1/module_tests/__init__.py @@ -1,12 +0,0 @@ -from pathlib import Path - -from bbot.modules import module_loader - -parent_dir = Path(__file__).parent - -module_test_files = list(parent_dir.glob("test_module_*.py")) -module_test_files = [m.name.split("test_module_")[-1].split(".")[0] for m in module_test_files] - -for module_name in module_loader.preloaded(): - module_name = module_name.lower() - assert module_name in module_test_files, f'No test file found for module "{module_name}"' From a2943d858eb68d122b0906618bc95f9ab6f317eb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 20 May 2023 00:00:42 -0400 Subject: [PATCH 030/387] gowitness and github tests --- bbot/modules/github.py | 4 +- bbot/test/conftest.py | 2 +- bbot/test/modules_test_classes.py | 38 ------- .../module_tests/test_module_github.py | 100 ++++++++++++++++++ .../module_tests/test_module_gowitness.py | 42 ++++++++ 5 files changed, 145 insertions(+), 41 deletions(-) diff --git a/bbot/modules/github.py b/bbot/modules/github.py index 9fddf0393e..f6a933ea50 100644 --- a/bbot/modules/github.py +++ b/bbot/modules/github.py @@ -42,7 +42,7 @@ async def query(self, query): try: async for r in agen: if r is None: - continue + break status_code = getattr(r, "status_code", 0) if status_code == 429: "Github is rate-limiting us (HTTP status: 429)" @@ -51,7 +51,7 @@ async def query(self, query): j = r.json() except Exception as e: self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") - continue + break items = j.get("items", []) if not items: break diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 0aca0ff1bd..c3cdb78766 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -14,7 +14,7 @@ def pytest_sessionfinish(session, exitstatus): logger.removeHandler(handler) # Wipe out BBOT home dir - shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) + # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py index a4a7665ebf..e6ac6b67fe 100644 --- a/bbot/test/modules_test_classes.py +++ b/bbot/test/modules_test_classes.py @@ -34,44 +34,6 @@ def check_events(self, events): assert open_port, "Failed to visit target OPEN_TCP_PORT" -class Gowitness(HttpxMockHelper): - additional_modules = ["httpx"] - import shutil - from pathlib import Path - - home_dir = Path("/tmp/.bbot_gowitness_test") - shutil.rmtree(home_dir, ignore_errors=True) - config_overrides = {"force_deps": True, "home": str(home_dir)} - - def mock_args(self): - respond_args = { - "response_data": """BBOT is life - - - -""", - "headers": {"Server": "Apache/2.4.41 (Ubuntu)"}, - } - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - screenshots_path = self.home_dir / "scans" / "gowitness_test" / "gowitness" / "screenshots" - screenshots = list(screenshots_path.glob("*.png")) - assert screenshots, f"No .png files found at {screenshots_path}" - url = False - webscreenshot = False - technology = False - for event in events: - if event.type == "URL_UNVERIFIED": - url = True - elif event.type == "WEBSCREENSHOT": - webscreenshot = True - elif event.type == "TECHNOLOGY": - technology = True - assert url, "No URL emitted" - assert webscreenshot, "No WEBSCREENSHOT emitted" - assert technology, "No TECHNOLOGY emitted" - class Subdomain_Hijack(HttpxMockHelper): additional_modules = ["httpx", "excavate"] diff --git a/bbot/test/test_step_1/module_tests/test_module_github.py b/bbot/test/test_step_1/module_tests/test_module_github.py index e69de29bb2..52f8154e9f 100644 --- a/bbot/test/test_step_1/module_tests/test_module_github.py +++ b/bbot/test/test_step_1/module_tests/test_module_github.py @@ -0,0 +1,100 @@ +from .base import ModuleTestBase + + +class TestGithub(ModuleTestBase): + config_overrides = {"omit_event_types": [], "scope_report_distance": 1} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response(url="https://api.github.com/zen") + module_test.httpx_mock.add_response( + url="https://api.github.com/search/code?per_page=100&type=Code&q=blacklanternsecurity.com&page=1", + json={ + "total_count": 214, + "incomplete_results": False, + "items": [ + { + "name": "main.go", + "path": "v2/cmd/cve-annotate/main.go", + "sha": "4aa7c9ec68acb4c603d4b9163bf7ed42de1939fe", + "url": "https://api.github.com/repositories/252813491/contents/v2/cmd/cve-annotate/main.go?ref=06f242e5fce3439b7418877676810cbf57934875", + "git_url": "https://api.github.com/repositories/252813491/git/blobs/4aa7c9ec68acb4c603d4b9163bf7ed42de1939fe", + "html_url": "https://github.com/projectdiscovery/nuclei/blob/06f242e5fce3439b7418877676810cbf57934875/v2/cmd/cve-annotate/main.go", + "repository": { + "id": 252813491, + "node_id": "MDEwOlJlcG9zaXRvcnkyNTI4MTM0OTE=", + "name": "nuclei", + "full_name": "projectdiscovery/nuclei", + "private": False, + "owner": { + "login": "projectdiscovery", + "id": 50994705, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjUwOTk0NzA1", + "avatar_url": "https://avatars.githubusercontent.com/u/50994705?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/projectdiscovery", + "html_url": "https://github.com/projectdiscovery", + "followers_url": "https://api.github.com/users/projectdiscovery/followers", + "following_url": "https://api.github.com/users/projectdiscovery/following{/other_user}", + "gists_url": "https://api.github.com/users/projectdiscovery/gists{/gist_id}", + "starred_url": "https://api.github.com/users/projectdiscovery/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/projectdiscovery/subscriptions", + "organizations_url": "https://api.github.com/users/projectdiscovery/orgs", + "repos_url": "https://api.github.com/users/projectdiscovery/repos", + "events_url": "https://api.github.com/users/projectdiscovery/events{/privacy}", + "received_events_url": "https://api.github.com/users/projectdiscovery/received_events", + "type": "Organization", + "site_admin": False, + }, + "html_url": "https://github.com/projectdiscovery/nuclei", + "description": "Fast and customizable vulnerability scanner based on simple YAML based DSL.", + "fork": False, + "url": "https://api.github.com/repos/projectdiscovery/nuclei", + "forks_url": "https://api.github.com/repos/projectdiscovery/nuclei/forks", + "keys_url": "https://api.github.com/repos/projectdiscovery/nuclei/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/projectdiscovery/nuclei/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/projectdiscovery/nuclei/teams", + "hooks_url": "https://api.github.com/repos/projectdiscovery/nuclei/hooks", + "issue_events_url": "https://api.github.com/repos/projectdiscovery/nuclei/issues/events{/number}", + "events_url": "https://api.github.com/repos/projectdiscovery/nuclei/events", + "assignees_url": "https://api.github.com/repos/projectdiscovery/nuclei/assignees{/user}", + "branches_url": "https://api.github.com/repos/projectdiscovery/nuclei/branches{/branch}", + "tags_url": "https://api.github.com/repos/projectdiscovery/nuclei/tags", + "blobs_url": "https://api.github.com/repos/projectdiscovery/nuclei/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/projectdiscovery/nuclei/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/projectdiscovery/nuclei/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/projectdiscovery/nuclei/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/projectdiscovery/nuclei/statuses/{sha}", + "languages_url": "https://api.github.com/repos/projectdiscovery/nuclei/languages", + "stargazers_url": "https://api.github.com/repos/projectdiscovery/nuclei/stargazers", + "contributors_url": "https://api.github.com/repos/projectdiscovery/nuclei/contributors", + "subscribers_url": "https://api.github.com/repos/projectdiscovery/nuclei/subscribers", + "subscription_url": "https://api.github.com/repos/projectdiscovery/nuclei/subscription", + "commits_url": "https://api.github.com/repos/projectdiscovery/nuclei/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/projectdiscovery/nuclei/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/projectdiscovery/nuclei/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/projectdiscovery/nuclei/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/projectdiscovery/nuclei/contents/{+path}", + "compare_url": "https://api.github.com/repos/projectdiscovery/nuclei/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/projectdiscovery/nuclei/merges", + "archive_url": "https://api.github.com/repos/projectdiscovery/nuclei/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/projectdiscovery/nuclei/downloads", + "issues_url": "https://api.github.com/repos/projectdiscovery/nuclei/issues{/number}", + "pulls_url": "https://api.github.com/repos/projectdiscovery/nuclei/pulls{/number}", + "milestones_url": "https://api.github.com/repos/projectdiscovery/nuclei/milestones{/number}", + "notifications_url": "https://api.github.com/repos/projectdiscovery/nuclei/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/projectdiscovery/nuclei/labels{/name}", + "releases_url": "https://api.github.com/repos/projectdiscovery/nuclei/releases{/id}", + "deployments_url": "https://api.github.com/repos/projectdiscovery/nuclei/deployments", + }, + "score": 1.0, + } + ], + }, + ) + + def check(self, module_test, events): + assert any( + e.data + == "https://raw.githubusercontent.com/projectdiscovery/nuclei/06f242e5fce3439b7418877676810cbf57934875/v2/cmd/cve-annotate/main.go" + for e in events + ), "Failed to detect URL" diff --git a/bbot/test/test_step_1/module_tests/test_module_gowitness.py b/bbot/test/test_step_1/module_tests/test_module_gowitness.py index e69de29bb2..278eb21a3f 100644 --- a/bbot/test/test_step_1/module_tests/test_module_gowitness.py +++ b/bbot/test/test_step_1/module_tests/test_module_gowitness.py @@ -0,0 +1,42 @@ +from .base import ModuleTestBase + + +class TestGowitness(ModuleTestBase): + targets = ["127.0.0.1:8888"] + modules_overrides = ["gowitness", "httpx"] + import shutil + from pathlib import Path + + home_dir = Path("/tmp/.bbot_gowitness_test") + shutil.rmtree(home_dir, ignore_errors=True) + config_overrides = {"force_deps": True, "home": str(home_dir)} + + def setup_after_prep(self, module_test): + respond_args = { + "response_data": """BBOT is life + + + +""", + "headers": {"Server": "Apache/2.4.41 (Ubuntu)"}, + } + module_test.set_expect_requests(respond_args=respond_args) + + + def check(self, module_test, events): + screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots" + screenshots = list(screenshots_path.glob("*.png")) + assert screenshots, f"No .png files found at {screenshots_path}" + url = False + webscreenshot = False + technology = False + for event in events: + if event.type == "URL_UNVERIFIED": + url = True + elif event.type == "WEBSCREENSHOT": + webscreenshot = True + elif event.type == "TECHNOLOGY": + technology = True + assert url, "No URL emitted" + assert webscreenshot, "No WEBSCREENSHOT emitted" + assert technology, "No TECHNOLOGY emitted" From 906d944198cfec136bd8e90d527c5cea702e64ca Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 20 May 2023 12:13:41 -0400 Subject: [PATCH 031/387] more module tests --- bbot/modules/hackertarget.py | 6 +- bbot/modules/hunterio.py | 30 +- bbot/test/bbot_fixtures.py | 13 + bbot/test/conftest.py | 2 +- bbot/test/modules_test_classes.py | 1377 ----------------- .../module_tests/test_module_generic_ssrf.py | 6 + .../module_tests/test_module_gowitness.py | 1 - .../module_tests/test_module_hackertarget.py | 13 + .../module_tests/test_module_host_header.py | 6 + .../module_tests/test_module_http.py | 24 + .../module_tests/test_module_httpx.py | 29 + .../module_tests/test_module_human.py | 8 + .../module_tests/test_module_hunt.py | 17 + .../module_tests/test_module_hunterio.py | 96 ++ .../test_module_iis_shortnames.py | 56 + bbot/test/test_step_1/test_modules_full.py | 234 --- bbot/test/test_step_2/test_scope.py | 46 +- bbot/test/test_step_2/test_web.py | 5 + 18 files changed, 304 insertions(+), 1665 deletions(-) delete mode 100644 bbot/test/modules_test_classes.py delete mode 100644 bbot/test/test_step_1/test_modules_full.py diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py index 38ff695818..8392110340 100644 --- a/bbot/modules/hackertarget.py +++ b/bbot/modules/hackertarget.py @@ -9,8 +9,10 @@ class hackertarget(crobat): base_url = "https://api.hackertarget.com" - def request_url(self, query): - return self.request_with_fail_count(f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}") + async def request_url(self, query): + url = f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}" + response = await self.request_with_fail_count(url) + return response def parse_results(self, r, query): for line in r.text.splitlines(): diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 845488844c..8bb9f74744 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -10,19 +10,17 @@ class hunterio(shodan_dns): options_desc = {"api_key": "Hunter.IO API key"} base_url = "https://api.hunter.io/v2" + limit = 100 - def setup(self): - self.limit = 100 - return super().setup() - - def ping(self): - r = self.helpers.request(f"{self.base_url}/account?api_key={self.api_key}") + async def ping(self): + url = f"{self.base_url}/account?api_key={self.api_key}" + r = await self.helpers.request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - for entry in self.query(query): + for entry in await self.query(query): email = entry.get("value", "") sources = entry.get("sources", []) if email: @@ -37,15 +35,19 @@ def handle_event(self, event): if url: self.emit_event(url, "URL_UNVERIFIED", email_event) - def query(self, query): + async def query(self, query): emails = [] url = ( f"{self.base_url}/domain-search?domain={query}&api_key={self.api_key}" + "&limit={page_size}&offset={offset}" ) - for j in self.helpers.api_page_iter(url, page_size=self.limit): - new_emails = j.get("data", {}).get("emails", []) - if not new_emails: - break - emails += new_emails + agen = self.helpers.api_page_iter(url, page_size=self.limit) + try: + async for j in agen: + new_emails = j.get("data", {}).get("emails", []) + if not new_emails: + break + emails += new_emails + finally: + agen.aclose() return emails diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 2b0553475a..f30710b5b0 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -336,3 +336,16 @@ def install_all_python_deps(): for module in module_loader.preloaded().values(): deps_pip.update(set(module.get("deps", {}).get("pip", []))) subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) + + +def tempwordlist(content): + tmp_path = "/tmp/.bbot_test/" + from bbot.core.helpers.misc import rand_string, mkdir + + mkdir(tmp_path) + filename = f"{tmp_path}{rand_string(8)}" + with open(filename, "w", errors="ignore") as f: + for c in content: + line = f"{c}\n" + f.write(line) + return filename diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index c3cdb78766..0aca0ff1bd 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -14,7 +14,7 @@ def pytest_sessionfinish(session, exitstatus): logger.removeHandler(handler) # Wipe out BBOT home dir - # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) + shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py deleted file mode 100644 index e6ac6b67fe..0000000000 --- a/bbot/test/modules_test_classes.py +++ /dev/null @@ -1,1377 +0,0 @@ -import re -import json -import logging - -from .helpers import * - -log = logging.getLogger(f"bbot.test") - - -class Httpx(HttpxMockHelper): - targets = ["http://127.0.0.1:8888/url", "127.0.0.1:8888"] - - def mock_args(self): - request_args = dict(uri="/", headers={"test": "header"}) - respond_args = dict(response_data=json.dumps({"open": "port"})) - self.set_expect_requests(request_args, respond_args) - request_args = dict(uri="/url", headers={"test": "header"}) - respond_args = dict(response_data=json.dumps({"url": "url"})) - self.set_expect_requests(request_args, respond_args) - - def check_events(self, events): - url = False - open_port = False - for e in events: - if e.type == "HTTP_RESPONSE": - j = json.loads(e.data["body"]) - if e.data["path"] == "/": - if j.get("open", "") == "port": - open_port = True - elif e.data["path"] == "/url": - if j.get("url", "") == "url": - url = True - assert url, "Failed to visit target URL" - assert open_port, "Failed to visit target OPEN_TCP_PORT" - - - -class Subdomain_Hijack(HttpxMockHelper): - additional_modules = ["httpx", "excavate"] - - def mock_args(self): - fingerprints = self.module.fingerprints - assert fingerprints, "No subdomain hijacking fingerprints available" - fingerprint = next(iter(fingerprints)) - rand_string = self.scan.helpers.rand_string(length=15, digits=False) - self.rand_subdomain = f"{rand_string}.{next(iter(fingerprint.domains))}" - respond_args = {"response_data": f''} - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - assert any( - event.type == "FINDING" - and event.data["description"].startswith("Hijackable Subdomain") - and self.rand_subdomain in event.data["description"] - and event.data["host"] == self.rand_subdomain - for event in events - ), f"No hijackable subdomains in {events}" - - -class Fingerprintx(HttpxMockHelper): - targets = ["127.0.0.1:8888"] - - def mock_args(self): - pass - - def check_events(self, events): - assert any( - event.type == "PROTOCOL" - and event.host == self.scan.helpers.make_ip_type("127.0.0.1") - and event.port == 8888 - and event.data["protocol"] == "HTTP" - for event in events - ), "HTTP protocol not detected" - - -class Otx(RequestMockHelper): - def mock_args(self): - for t in self.targets: - self.httpx_mock.add_response( - url=f"https://otx.alienvault.com/api/v1/indicators/domain/{t}/passive_dns", - json={ - "passive_dns": [ - { - "address": "2606:50c0:8000::153", - "first": "2021-10-28T20:23:08", - "last": "2022-08-24T18:29:49", - "hostname": "asdf.blacklanternsecurity.com", - "record_type": "AAAA", - "indicator_link": "/indicator/hostname/www.blacklanternsecurity.com", - "flag_url": "assets/images/flags/us.png", - "flag_title": "United States", - "asset_type": "hostname", - "asn": "AS54113 fastly", - } - ] - }, - ) - - def check_events(self, events): - assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" - - -class Anubisdb(RequestMockHelper): - def setup(self, scan): - self.module.abort_if = lambda e: False - - def mock_args(self): - for t in self.targets: - self.httpx_mock.add_response( - url=f"https://jldc.me/anubis/subdomains/{t}", - json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], - ) - - def check_events(self, events): - assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" - - -class SecretsDB(HttpxMockHelper): - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "-----BEGIN PGP PRIVATE KEY BLOCK-----"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "FINDING" for e in events) - - -class Badsecrets(HttpxMockHelper): - targets = [ - "http://127.0.0.1:8888/", - "http://127.0.0.1:8888/test.aspx", - "http://127.0.0.1:8888/cookie.aspx", - "http://127.0.0.1:8888/cookie2.aspx", - ] - - sample_viewstate = """ -

-
- -
- -
- - - -
- - - -""" - - sample_viewstate_notvuln = """ -
-
- -
- -
- - - -
-
- - -""" - - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"uri": "/test.aspx"} - respond_args = {"response_data": self.sample_viewstate} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - respond_args = {"response_data": self.sample_viewstate_notvuln} - self.set_expect_requests(respond_args=respond_args) - - expect_args = {"uri": "/cookie.aspx"} - respond_args = { - "response_data": "

JWT Cookie Test

", - "headers": { - "set-cookie": "vulnjwt=eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo; secure" - }, - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"uri": "/cookie2.aspx"} - respond_args = { - "response_data": "

Express Cookie Test

", - "headers": { - "set-cookie": "connect.sid=s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc; Path=/; Expires=Wed, 05 Apr 2023 04:47:29 GMT; HttpOnly" - }, - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - SecretFound = False - IdentifyOnly = False - CookieBasedDetection = False - CookieBasedDetection_2 = False - - for e in events: - if ( - e.type == "VULNERABILITY" - and e.data["description"] - == "Known Secret Found. Secret Type: [ASP.NET MachineKey] Secret: [validationKey: 0F97BAE23F6F36801ABDB5F145124E00A6F795A97093D778EE5CD24F35B78B6FC4C0D0D4420657689C4F321F8596B59E83F02E296E970C4DEAD2DFE226294979 validationAlgo: SHA1 encryptionKey: 8CCFBC5B7589DD37DC3B4A885376D7480A69645DAEEC74F418B4877BEC008156 encryptionAlgo: AES] Product Type: [ASP.NET Viewstate] Product: [rJdyYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESf4nBu] Detecting Module: [ASPNET_Viewstate]" - ): - SecretFound = True - - if ( - e.type == "FINDING" - and e.data["description"] - == "Cryptographic Product identified. Product Type: [ASP.NET Viewstate] Product: [AAAAYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESfAAAA] Detecting Module: [ASPNET_Viewstate]" - ): - IdentifyOnly = True - - if ( - e.type == "VULNERABILITY" - and e.data["description"] - == "Known Secret Found. Secret Type: [HMAC/RSA Key] Secret: [1234] Product Type: [JSON Web Token (JWT)] Product: [eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo] Detecting Module: [Generic_JWT]" - ): - CookieBasedDetection = True - - if ( - e.type == "VULNERABILITY" - and e.data["description"] - == "Known Secret Found. Secret Type: [Express.js SESSION_SECRET] Secret: [keyboard cat] Product Type: [Express.js Signed Cookie] Product: [s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc] Detecting Module: [ExpressSignedCookies]" - ): - CookieBasedDetection_2 = True - - assert SecretFound, "No secret found" - assert IdentifyOnly, "No crypto product identified" - assert CookieBasedDetection, "No JWT cookie detected" - assert CookieBasedDetection_2, "No Express.js cookie detected" - - -class Telerik(HttpxMockHelper): - additional_modules = ["httpx"] - config_overrides = {"modules": {"telerik": {"exploit_RAU_crypto": True}}} - - def mock_args(self): - # Simulate Telerik.Web.UI.WebResource.axd?type=rau detection - expect_args = {"method": "GET", "uri": "/Telerik.Web.UI.WebResource.axd", "query_string": "type=rau"} - respond_args = { - "response_data": '{ "message" : "RadAsyncUpload handler is registered succesfully, however, it may not be accessed directly." }' - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # Simulate Vulnerable Telerik.Web.UI.WebResource.axd - vuln_data = "ATTu5i4R+ViNFYO6kst0jC11wM/1iqH+W/isjhaDjNuCI7eJ/BY5d1E9eqZK27CJCMuon9u8/hgRIM/cTlgLlv4qOYjPBjs81Y3dAZAdtIr3TXiCmZi9M09a1BYMxjvGKfVky3b7PoOppeWS/3rglTwL1e8oyqLGx2NKUH5y8Cd+kLKV2f31J1sV4I5HTDKgDmvziJp3zlDrCb0Fi9ilKH+O1cbVx6SdBop/U30FxLaB/QIbt2N1rQHREJ5Skpgo7dilPxzBaTObdBhCVyB/FiJhenS/0u3h0Mpi6+A40SylICcyyxQha7+Uh7lEJ8Ne+2eTs4WqcaaQbvIhy7oHc+D0soxRKMZRjo7Up+UWHQJJh6KtWSCxUESNSdNcxjPQZE9HqsPlldVlkeC+ehSGce5bR0Ylots6Iz1OoCgMEWwxByeG3VzgxF6XpitL61A1hFcNo9euSTnCfOWh0vrQHON7DN5LpM9xr7SoD0Dnu01hZ9NS1PHhPLyN5WS87u5qdZp/z3Sxwc3wawIdo62RNf4Iz2gAKJZnPfxrE1mRn5kBe7f6O44rcuv6lcdao/DGlwbERKwRI6/n+FxGmc7H5iEKyihIwS2XUoOgsYTx5CWCDM8CuOXTk+H5fPYp9APRPbkD1IS9I/vRmvNPwWsgv8/7DzttqdBsGxiZJfCw1uZ7KSVmbItgXPAcscNxGEMaHXyJzkAl/mlM5/t/YSejwYoSW6jFfQcLdaVx2dpIpl5UmmQjFedzKeiNqpZDCk4yzXFHX24XUODYMJDtIJK2Hz1KTZmFG+LAOJjB9QOI58hFAnytcKay+JWFrzah/IvoNZxJUtlYdxw0YEyKs/ExET7AXgYQN0S+8j2PfaMMpzDSctTqpp5XBFV4Mt718GiqVnQJtWQv2p9Xl8XXOerBthbzzAciVcB8AV2WfZ51W3e4aX4kcyT/sCJhm7NR5WrNG5mX/ns0TTnGnzlPYhJcbu8uMFjMGDpXuhVyroJ7wmZucaIvesg0h5Y9cMEFviqsdy15vjMzFh+v9uO9Vicf6n9Z9JGSpWKE8wer2JU5b53Zw0cTfulAAffLWXnzOnfu&6R/cGaqQeHVAzdJ9wTFOyCsrMSTtqcjLe8AHwiPckPDUwecnJyNlkDYwDQpxGYQ9hs6YxhupK310sbCbtXB4H6Dz5rGNL40nkkyo4j2clmRr08jtFsPQ0RpE5BGsulPT3l0MxyAvPFMs8bMybUyAP+9RB9LoHE3Xo8BqDadX3HQakpPfGtiDMp+wxkWRgaNpCnXeY1QewWTF6z/duLzbu6CT6s+H4HgBHrOLTpemC2PvP2bDm0ySPHLdpapLYxU8nIYjLKIyYJgwv9S9jNckIVpcGVTWVul7CauCKxAB2mMnM9jJi8zfFwKajT5d2d9XfpkiVMrdlmikSB/ehyX1wQ==" - expect_args = { - "method": "POST", - "uri": "/Telerik.Web.UI.WebResource.axd", - "query_string": "type=rau", - "data": vuln_data, - } - respond_args = { - "response_data": '{"fileInfo":{"FileName":"RAU_crypto.bypass","ContentType":"text/html","ContentLength":5,"DateJson":"2019-01-02T03:04:05.067Z","Index":0}, "metaData":"CS8S/Z0J/b2982DRxDin0BBslA7fI0cWMuWlPu4W3FkE4tKaVoIEiAOtVlJ6D+0RQsfu8ox6gvMYxceQ0LtWyTkQBaIUa8LgLQg05DMaQuufHNx0YQ2ACi5neqDBvduj2MGiSGC0hNKzSWsHystZGUfFPLTZuJXYnff+WXurecuRzSI7d4Q1aj0bcTKKvfyQtH+fsTEafWRRZ99X/xgi4ON2OsRZ738uQHw7pQT2e1v7AtN46mxO/BmhEuZQr6m6HEvxK0pJRNkBhFUiQ+poeu8j3JzicOjvPDwFE4Rjqf3RVILt83XZrju2VpRIJqAEtf//znhH8BhT5BWvhnRo+J3ML5qoZLa2joE/QK8Ctf3UPvAFkHIUMdOH2mLNgZ+U87tdVE6fYfzvphZsLxmJRG45H8ZTZuYhJbOfei2LQ4fqHmr7p8KpJNVqoz/ev1dnBclAf5ayb40qJKEVsGXIbWEbIZwg7TTsLFc29aP7DPg=" }' - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # Simulate DialogHandler detection - expect_args = {"method": "GET", "uri": "Telerik.Web.UI.SpellCheckHandler.axd"} - respond_args = {"status": 500} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # Simulate DialogHandler detection - expect_args = {"method": "GET", "uri": "/App_Master/Telerik.Web.UI.DialogHandler.aspx"} - respond_args = { - "response_data": '
Cannot deserialize dialog parameters. Please refresh the editor page.
Error Message:Invalid length for a Base-64 char array or string.
' - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # Fallback - expect_args = {"uri": re.compile(r"^/\w{10}$")} - respond_args = {"status": 200} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - telerik_axd_detection = False - telerik_axd_vulnerable = False - telerik_spellcheck_detection = False - telerik_dialoghandler_detection = False - - for e in events: - if e.type == "FINDING" and "Telerik RAU AXD Handler detected" in e.data["description"]: - telerik_axd_detection = True - continue - - if e.type == "VULNERABILITY" and "Confirmed Vulnerable Telerik (version: 2014.3.1024)": - telerik_axd_vulnerable = True - continue - - if e.type == "FINDING" and "Telerik DialogHandler detected" in e.data["description"]: - telerik_dialoghandler_detection = True - continue - - if e.type == "FINDING" and "Telerik SpellCheckHandler detected" in e.data["description"]: - telerik_spellcheck_detection = True - continue - - assert telerik_axd_detection, "Telerik AXD detection failed" - assert telerik_axd_vulnerable, "Telerik vulnerable AXD detection failed" - assert telerik_spellcheck_detection, "Telerik spellcheck detection failed" - assert telerik_dialoghandler_detection, "Telerik dialoghandler detection failed" - - -class Paramminer_headers(HttpxMockHelper): - headers_body = """ - - the title - -

Hello null!

'; - - - """ - - headers_body_match = """ - - the title - -

Hello AAAAAAAAAAAAAA!

'; - - - """ - additional_modules = ["httpx"] - - config_overrides = {"modules": {"paramminer_headers": {"wordlist": tempwordlist(["junkword1", "tracestate"])}}} - - def setup(self, scan): - from bbot.core.helpers import helper - - self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" - helper.HttpCompare.gen_cache_buster = lambda *args, **kwargs: {"AAAAAA": "1"} - - def mock_args(self): - expect_args = dict(headers={"tracestate": "AAAAAAAAAAAAAA"}) - respond_args = {"response_data": self.headers_body_match} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - respond_args = {"response_data": self.headers_body} - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [tracestate] Reasons: [body]" - for e in events - ) - assert not any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [junkword1] Reasons: [body]" - for e in events - ) - - -class Paramminer_getparams(HttpxMockHelper): - getparam_body = """ - - the title - -

Hello null!

'; - - - """ - - getparam_body_match = """ - - the title - -

Hello AAAAAAAAAAAAAA!

'; - - - """ - additional_modules = ["httpx"] - - config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist(["canary", "id"])}}} - - def setup(self, scan): - from bbot.core.helpers import helper - - self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" - helper.HttpCompare.gen_cache_buster = lambda *args, **kwargs: {"AAAAAA": "1"} - - def mock_args(self): - expect_args = {"query_string": b"id=AAAAAAAAAAAAAA&AAAAAA=1"} - respond_args = {"response_data": self.getparam_body_match} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - respond_args = {"response_data": self.getparam_body} - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [id] Reasons: [body]" - for e in events - ) - assert not any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [canary] Reasons: [body]" - for e in events - ) - - -class Paramminer_cookies(HttpxMockHelper): - cookies_body = """ - - the title - -

Hello null!

'; - - - """ - - cookies_body_match = """ - - the title - -

Hello AAAAAAAAAAAAAA!

'; - - - """ - additional_modules = ["httpx"] - - config_overrides = {"modules": {"paramminer_cookies": {"wordlist": tempwordlist(["junkcookie", "admincookie"])}}} - - def setup(self, scan): - from bbot.core.helpers import helper - - self.module.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" - helper.HttpCompare.gen_cache_buster = lambda *args, **kwargs: {"AAAAAA": "1"} - - def mock_args(self): - expect_args = dict(headers={"Cookie": "admincookie=AAAAAAAAAAAAAA"}) - respond_args = {"response_data": self.cookies_body_match} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - respond_args = {"response_data": self.cookies_body} - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [admincookie] Reasons: [body]" - for e in events - ) - assert not any( - e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [junkcookie] Reasons: [body]" - for e in events - ) - - -class LeakIX(RequestMockHelper): - def mock_args(self): - self.httpx_mock.add_response( - url="https://leakix.net/api/subdomains/blacklanternsecurity.com", - json=[ - { - "subdomain": "www.blacklanternsecurity.com", - "distinct_ips": 2, - "last_seen": "2023-02-20T20:23:13.583Z", - }, - { - "subdomain": "asdf.blacklanternsecurity.com", - "distinct_ips": 1, - "last_seen": "2022-09-17T01:31:52.563Z", - }, - ], - ) - - def check_events(self, events): - www = False - asdf = False - for e in events: - if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "leakix": - if e.data == "www.blacklanternsecurity.com": - www = True - elif e.data == "asdf.blacklanternsecurity.com": - asdf = True - assert www - assert asdf - - -class Massdns(RequestMockHelper): - subdomain_wordlist = tempwordlist(["www", "asdf"]) - config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist)}}} - - def __init__(self, request): - super().__init__(request) - self.httpx_mock.add_response( - url="https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt", - text="8.8.8.8\n8.8.4.4\n1.1.1.1", - ) - - def mock_args(self): - pass - - def check_events(self, events): - assert any( - e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and e.data == "www.blacklanternsecurity.com" for e in events - ) - - -class Robots(HttpxMockHelper): - additional_modules = ["httpx"] - - config_overrides = {"modules": {"robots": {"include_sitemap": True}}} - - def mock_args(self): - sample_robots = f"Allow: /allow/\nDisallow: /disallow/\nJunk: test.com\nDisallow: /*/wildcard.txt\nSitemap: {self.targets[0]}sitemap.txt" - - expect_args = {"method": "GET", "uri": "/robots.txt"} - respond_args = {"response_data": sample_robots} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - allow_bool = False - disallow_bool = False - sitemap_bool = False - wildcard_bool = False - - for e in events: - if e.type == "URL_UNVERIFIED": - if str(e.module) != "TARGET": - assert "spider-danger" in e.tags, f"{e} doesn't have spider-danger tag" - if e.data == "http://127.0.0.1:8888/allow/": - allow_bool = True - - if e.data == "http://127.0.0.1:8888/disallow/": - disallow_bool = True - - if e.data == "http://127.0.0.1:8888/sitemap.txt": - sitemap_bool = True - - if re.match(r"http://127\.0\.0\.1:8888/\w+/wildcard\.txt", e.data): - wildcard_bool = True - - assert allow_bool - assert disallow_bool - assert sitemap_bool - assert wildcard_bool - - -class Masscan(MockHelper): - targets = ["8.8.8.8/32"] - config_overrides = {"modules": {"masscan": {"ports": "443", "wait": 1}}} - config_overrides_2 = {"modules": {"masscan": {"ports": "443", "wait": 1, "use_cache": True}}} - masscan_output = """[ -{ "ip": "8.8.8.8", "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] } -]""" - masscan_config = """seed = 17230484647655100360 -rate = 600 -shard = 1/1 - - -# TARGET SELECTION (IP, PORTS, EXCLUDES) -ports = -range = 9.8.7.6""" - - def __init__(self, request): - super().__init__(request) - config2 = OmegaConf.merge(self.config, OmegaConf.create(self.config_overrides_2)) - self.add_scan( - *self.targets, - modules=[self.name] + self.additional_modules, - name=f"{self.name}_test", - config=config2, - whitelist=self.whitelist, - blacklist=self.blacklist, - ) - self.masscan_run = False - - async def run_masscan(self, command, *args, **kwargs): - if "masscan" in command[:2]: - for l in self.masscan_output.splitlines(): - yield l - self.masscan_run = True - else: - async for l in self.scan.helpers.run_live(command, *args, **kwargs): - yield l - - def setup(self, scan): - scan.helpers.run_live = self.run_masscan - scan.modules["masscan"].masscan_config = self.masscan_config - - async def run(self): - for i, scan in enumerate(self.scans): - await scan.prep() - self.setup(scan) - events = [e async for e in scan.start()] - self.check_events(events) - if i == 0: - assert self.masscan_run == True, "masscan didn't run when it was supposed to" - self.masscan_run = False - else: - assert self.masscan_run == False, "masscan ran when it wasn't supposed to" - - def check_events(self, events): - assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" - assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" - - -class Buckets(HttpxMockHelper): - providers = ["aws", "gcp", "azure", "digitalocean", "firebase"] - # providers = ["azure"] - additional_modules = ["excavate", "speculate", "httpx"] + [f"bucket_{p}" for p in providers] - config_overrides = { - "modules": { - "bucket_aws": {"permutations": True}, - "bucket_gcp": {"permutations": True}, - "bucket_azure": {"permutations": True}, - "bucket_digitalocean": {"permutations": True}, - "bucket_firebase": {"permutations": True}, - }, - "excavate": True, - "speculate": True, - } - - from bbot.core.helpers.misc import rand_string - - random_bucket_name_1 = rand_string(15, digits=False) - random_bucket_name_2 = rand_string(15, digits=False) - - open_aws_bucket = """ -vpn-static1000falsestyle.css2017-03-18T06:41:59.000Z"bf9e72bdab09b785f05ff0395023cc35"429STANDARD""" - open_digitalocean_bucket = """cloud011000falsetest.doc2020-10-14T15:23:37.545Z"4d25c8699f7347acc9f41e57148c62c0"13362425STANDARD19578831957883Normal""" - open_gcp_bucket = """{ - "kind": "storage#testIamPermissionsResponse", - "permissions": [ - "storage.objects.create", - "storage.objects.list" - ] -}""" - - def __init__(self, request, **kwargs): - self.httpx_mock = request.getfixturevalue("httpx_mock") - super().__init__(request, **kwargs) - - def setup(self, scan): - scan.helpers.word_cloud.mutations = lambda b, cloud=False: [ - (b, "dev"), - ] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - body = f""" -
- - - - - - - - - - - """ - self.set_expect_requests(expect_args=expect_args, respond_args={"response_data": body}) - - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}.s3-ap-southeast-2.amazonaws.com", - text=self.open_aws_bucket, - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}.fra1.digitaloceanspaces.com", - text=self.open_digitalocean_bucket, - ) - self.httpx_mock.add_response( - url=f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}/iam/testPermissions?permissions=storage.buckets.setIamPolicy&permissions=storage.objects.list&permissions=storage.objects.get&permissions=storage.objects.create", - text=self.open_gcp_bucket, - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}.firebaseio.com/.json", - text="", - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}-dev.s3.amazonaws.com", - text="", - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}-dev.fra1.digitaloceanspaces.com", - text="", - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}-dev.blob.core.windows.net/{self.random_bucket_name_2}-dev?restype=container", - text="", - ) - self.httpx_mock.add_response( - url=f"https://www.googleapis.com/storage/v1/b/{self.random_bucket_name_2}-dev", - text="", - ) - self.httpx_mock.add_response( - url=f"https://{self.random_bucket_name_2}-dev.firebaseio.com/.json", - text="", - ) - self.httpx_mock.add_response(url=re.compile(".*"), text="", status_code=404) - - def check_events(self, events): - for provider in self.providers: - # make sure buckets were excavated - assert any( - e.type == "STORAGE_BUCKET" and str(e.module) == f"{provider}_cloud" for e in events - ), f'bucket not found for provider "{provider}"' - # make sure open buckets were found - if not provider == "azure": - assert any( - e.type == "FINDING" and str(e.module) == f"bucket_{provider}" for e in events - ), f'open bucket not found for provider "{provider}"' - for e in events: - if e.type == "FINDING" and str(e.module) == f"bucket_{provider}": - url = e.data.get("url", "") - assert self.random_bucket_name_2 in url - assert not self.random_bucket_name_1 in url - assert not f"{self.random_bucket_name_2}-dev" in url - # make sure bucket mutations were found - assert any( - e.type == "STORAGE_BUCKET" - and str(e.module) == f"bucket_{provider}" - and f"{self.random_bucket_name_2}-dev" in e.data["url"] - for e in events - ), f'bucket (dev mutation) not found for provider "{provider}"' - - -class ASN(HttpxMockHelper): - targets = ["8.8.8.8"] - response_get_asn_ripe = { - "messages": [], - "see_also": [], - "version": "1.1", - "data_call_name": "network-info", - "data_call_status": "supported", - "cached": False, - "data": {"asns": ["15169"], "prefix": "8.8.8.0/24"}, - "query_id": "20230217212133-f278ff23-d940-4634-8115-a64dee06997b", - "process_time": 5, - "server_id": "app139", - "build_version": "live.2023.2.1.142", - "status": "ok", - "status_code": 200, - "time": "2023-02-17T21:21:33.428469", - } - response_get_asn_metadata_ripe = { - "messages": [], - "see_also": [], - "version": "4.1", - "data_call_name": "whois", - "data_call_status": "supported - connecting to ursa", - "cached": False, - "data": { - "records": [ - [ - {"key": "ASNumber", "value": "15169", "details_link": None}, - {"key": "ASName", "value": "GOOGLE", "details_link": None}, - {"key": "ASHandle", "value": "15169", "details_link": "https://stat.ripe.net/AS15169"}, - {"key": "RegDate", "value": "2000-03-30", "details_link": None}, - { - "key": "Ref", - "value": "https://rdap.arin.net/registry/autnum/15169", - "details_link": "https://rdap.arin.net/registry/autnum/15169", - }, - {"key": "source", "value": "ARIN", "details_link": None}, - ], - [ - {"key": "OrgAbuseHandle", "value": "ABUSE5250-ARIN", "details_link": None}, - {"key": "OrgAbuseName", "value": "Abuse", "details_link": None}, - {"key": "OrgAbusePhone", "value": "+1-650-253-0000", "details_link": None}, - { - "key": "OrgAbuseEmail", - "value": "network-abuse@google.com", - "details_link": "mailto:network-abuse@google.com", - }, - { - "key": "OrgAbuseRef", - "value": "https://rdap.arin.net/registry/entity/ABUSE5250-ARIN", - "details_link": "https://rdap.arin.net/registry/entity/ABUSE5250-ARIN", - }, - {"key": "source", "value": "ARIN", "details_link": None}, - ], - [ - {"key": "OrgName", "value": "Google LLC", "details_link": None}, - {"key": "OrgId", "value": "GOGL", "details_link": None}, - {"key": "Address", "value": "1600 Amphitheatre Parkway", "details_link": None}, - {"key": "City", "value": "Mountain View", "details_link": None}, - {"key": "StateProv", "value": "CA", "details_link": None}, - {"key": "PostalCode", "value": "94043", "details_link": None}, - {"key": "Country", "value": "US", "details_link": None}, - {"key": "RegDate", "value": "2000-03-30", "details_link": None}, - { - "key": "Comment", - "value": "Please note that the recommended way to file abuse complaints are located in the following links.", - "details_link": None, - }, - { - "key": "Comment", - "value": "To report abuse and illegal activity: https://www.google.com/contact/", - "details_link": None, - }, - { - "key": "Comment", - "value": "For legal requests: http://support.google.com/legal", - "details_link": None, - }, - {"key": "Comment", "value": "Regards,", "details_link": None}, - {"key": "Comment", "value": "The Google Team", "details_link": None}, - { - "key": "Ref", - "value": "https://rdap.arin.net/registry/entity/GOGL", - "details_link": "https://rdap.arin.net/registry/entity/GOGL", - }, - {"key": "source", "value": "ARIN", "details_link": None}, - ], - [ - {"key": "OrgTechHandle", "value": "ZG39-ARIN", "details_link": None}, - {"key": "OrgTechName", "value": "Google LLC", "details_link": None}, - {"key": "OrgTechPhone", "value": "+1-650-253-0000", "details_link": None}, - { - "key": "OrgTechEmail", - "value": "arin-contact@google.com", - "details_link": "mailto:arin-contact@google.com", - }, - { - "key": "OrgTechRef", - "value": "https://rdap.arin.net/registry/entity/ZG39-ARIN", - "details_link": "https://rdap.arin.net/registry/entity/ZG39-ARIN", - }, - {"key": "source", "value": "ARIN", "details_link": None}, - ], - [ - {"key": "RTechHandle", "value": "ZG39-ARIN", "details_link": None}, - {"key": "RTechName", "value": "Google LLC", "details_link": None}, - {"key": "RTechPhone", "value": "+1-650-253-0000", "details_link": None}, - {"key": "RTechEmail", "value": "arin-contact@google.com", "details_link": None}, - { - "key": "RTechRef", - "value": "https://rdap.arin.net/registry/entity/ZG39-ARIN", - "details_link": None, - }, - {"key": "source", "value": "ARIN", "details_link": None}, - ], - ], - "irr_records": [], - "authorities": ["arin"], - "resource": "15169", - "query_time": "2023-02-17T21:25:00", - }, - "query_id": "20230217212529-75f57efd-59f4-473f-8bdd-803062e94290", - "process_time": 268, - "server_id": "app143", - "build_version": "live.2023.2.1.142", - "status": "ok", - "status_code": 200, - "time": "2023-02-17T21:25:29.417812", - } - response_get_asn_bgpview = { - "status": "ok", - "status_message": "Query was successful", - "data": { - "ip": "8.8.8.8", - "ptr_record": "dns.google", - "prefixes": [ - { - "prefix": "8.8.8.0/24", - "ip": "8.8.8.0", - "cidr": 24, - "asn": {"asn": 15169, "name": "GOOGLE", "description": "Google LLC", "country_code": "US"}, - "name": "LVLT-GOGL-8-8-8", - "description": "Google LLC", - "country_code": "US", - } - ], - "rir_allocation": { - "rir_name": "ARIN", - "country_code": None, - "ip": "8.0.0.0", - "cidr": 9, - "prefix": "8.0.0.0/9", - "date_allocated": "1992-12-01 00:00:00", - "allocation_status": "allocated", - }, - "iana_assignment": { - "assignment_status": "legacy", - "description": "Administered by ARIN", - "whois_server": "whois.arin.net", - "date_assigned": None, - }, - "maxmind": {"country_code": None, "city": None}, - }, - "@meta": {"time_zone": "UTC", "api_version": 1, "execution_time": "567.18 ms"}, - } - response_get_emails_bgpview = { - "status": "ok", - "status_message": "Query was successful", - "data": { - "asn": 15169, - "name": "GOOGLE", - "description_short": "Google LLC", - "description_full": ["Google LLC"], - "country_code": "US", - "website": "https://about.google/intl/en/", - "email_contacts": ["network-abuse@google.com", "arin-contact@google.com"], - "abuse_contacts": ["network-abuse@google.com"], - "looking_glass": None, - "traffic_estimation": None, - "traffic_ratio": "Mostly Outbound", - "owner_address": ["1600 Amphitheatre Parkway", "Mountain View", "CA", "94043", "US"], - "rir_allocation": { - "rir_name": "ARIN", - "country_code": "US", - "date_allocated": "2000-03-30 00:00:00", - "allocation_status": "assigned", - }, - "iana_assignment": { - "assignment_status": None, - "description": None, - "whois_server": None, - "date_assigned": None, - }, - "date_updated": "2023-02-07 06:39:11", - }, - "@meta": {"time_zone": "UTC", "api_version": 1, "execution_time": "56.55 ms"}, - } - config_overrides = {"scope_report_distance": 2} - - def __init__(self, request, **kwargs): - super().__init__(request, **kwargs) - self.scan2 = self.add_scan( - *self.targets, - modules=[self.name] + self.additional_modules, - name=f"{self.name}_test_2", - config=self.config, - ) - - def mock_args(self): - self.httpx_mock.add_response( - url="https://stat.ripe.net/data/network-info/data.json?resource=8.8.8.8", - text=json.dumps(self.response_get_asn_ripe), - ) - self.httpx_mock.add_response( - url="https://stat.ripe.net/data/whois/data.json?resource=15169", - text=json.dumps(self.response_get_asn_metadata_ripe), - ) - self.httpx_mock.add_response( - url="https://api.bgpview.io/ip/8.8.8.8", text=json.dumps(self.response_get_asn_bgpview) - ) - self.httpx_mock.add_response( - url="https://api.bgpview.io/asn/15169", text=json.dumps(self.response_get_emails_bgpview) - ) - - async def run(self): - await self.scan.prep() - self.module.sources = ["bgpview", "ripe"] - events = [e async for e in self.scan.start() if e.module == self.module] - assert self.check_events(events) - await self.scan2.prep() - self.module2 = self.scan2.modules["asn"] - self.module2.sources = ["ripe", "bgpview"] - events2 = [e async for e in self.scan2.start() if e.module == self.module2] - assert self.check_events(events2) - - def check_events(self, events): - asn = False - email = False - for e in events: - if e.type == "ASN": - asn = True - elif e.type == "EMAIL_ADDRESS": - email = True - return asn and email - - -class Wafw00f(HttpxMockHelper): - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "Proudly powered by litespeed web server"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events) - - -class Vhost(HttpxMockHelper): - targets = ["http://localhost:8888", "secret.localhost"] - - additional_modules = ["httpx"] - - test_wordlist = ["11111111", "admin", "cloud", "junkword1", "zzzjunkword2"] - config_overrides = { - "modules": { - "vhost": { - "wordlist": tempwordlist(test_wordlist), - } - } - } - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "admin.localhost:8888"}} - respond_args = {"response_data": "Alive vhost admin"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "cloud.localhost:8888"}} - respond_args = {"response_data": "Alive vhost cloud"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "q-cloud.localhost:8888"}} - respond_args = {"response_data": "Alive vhost q-cloud"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "secret.localhost:8888"}} - respond_args = {"response_data": "Alive vhost secret"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "host.docker.internal"}} - respond_args = {"response_data": "Alive vhost host.docker.internal"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - basic_detection = False - mutaton_of_detected = False - basehost_mutation = False - special_vhost_list = False - wordcloud_detection = False - - for e in events: - if e.type == "VHOST": - if e.data["vhost"] == "admin": - basic_detection = True - if e.data["vhost"] == "cloud": - mutaton_of_detected = True - if e.data["vhost"] == "q-cloud": - basehost_mutation = True - if e.data["vhost"] == "host.docker.internal": - special_vhost_list = True - if e.data["vhost"] == "secret": - wordcloud_detection = True - - assert basic_detection - assert mutaton_of_detected - assert basehost_mutation - assert special_vhost_list - assert wordcloud_detection - - -class Iis_shortnames(HttpxMockHelper): - additional_modules = ["httpx"] - - config_overrides = {"modules": {"iis_shortnames": {"detect_only": False}}} - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 404 - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive", "status": 200} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/*~1*/a.aspx"} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/B\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/BL\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/BLS\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/BLSH\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/BLSHA\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": re.compile(r"\/BLSHAX\*~1\*.*$")} - respond_args = {"response_data": "", "status": 400} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - vulnerabilityEmitted = False - url_hintEmitted = False - for e in events: - if e.type == "VULNERABILITY": - vulnerabilityEmitted = True - if e.type == "URL_HINT" and e.data == "http://127.0.0.1:8888/BLSHAX~1": - url_hintEmitted = True - - assert vulnerabilityEmitted - assert url_hintEmitted - - -class Nuclei_manual(HttpxMockHelper): - additional_modules = ["httpx", "excavate"] - - test_html = """ - html> - - Index of /test - - -

Index of /test

- - - - -
NameLast modifiedSize
Parent Directory  -
-
Apache/2.4.38 (Debian) Server at http://127.0.0.1:8888/testmultipleruns.html
- -""" - config_overrides = { - "web_spider_distance": 1, - "web_spider_depth": 1, - "modules": { - "nuclei": { - "version": "2.9.4", - "mode": "manual", - "concurrency": 2, - "ratelimit": 10, - "templates": "/tmp/.bbot_test/tools/nuclei-templates/http/miscellaneous/", - "interactsh_disable": True, - "directory_only": False, - } - }, - } - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": self.test_html} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/testmultipleruns.html"} - respond_args = {"response_data": "Copyright 1984"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - first_run_detect = False - second_run_detect = False - for e in events: - if e.type == "FINDING": - if "Directory listing enabled" in e.data["description"]: - first_run_detect = True - elif "Copyright" in e.data["description"]: - second_run_detect = True - assert first_run_detect - assert second_run_detect - - -class Nuclei_severe(HttpxMockHelper): - additional_modules = ["httpx"] - - config_overrides = { - "modules": { - "nuclei": { - "mode": "severe", - "concurrency": 1, - "templates": "/tmp/.bbot_test/tools/nuclei-templates/vulnerabilities/generic/generic-linux-lfi.yaml", - } - }, - "interactsh_disable": True, - } - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/etc/passwd"} - respond_args = {"response_data": "root:.*:0:0:"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "VULNERABILITY" and "Generic Linux - Local File Inclusion" in e.data["description"] - for e in events - ) - - -class Nuclei_technology(HttpxMockHelper): - additional_modules = ["httpx"] - - config_overrides = { - "interactsh_disable": True, - "modules": {"nuclei": {"mode": "technology", "concurrency": 2, "tags": "apache"}}, - } - - def __init__(self, request, caplog, **kwargs): - self.caplog = caplog - super().__init__(request, **kwargs) - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = { - "response_data": "", - "headers": {"Server": "Apache/2.4.52 (Ubuntu)"}, - } - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - if "Using Interactsh Server" in self.caplog.text: - return False - assert any(e.type == "FINDING" and "apache" in e.data["description"] for e in events) - - -class Nuclei_budget(HttpxMockHelper): - additional_modules = ["httpx"] - - config_overrides = { - "modules": { - "nuclei": { - "mode": "budget", - "concurrency": 1, - "tags": "spiderfoot", - "templates": "/tmp/.bbot_test/tools/nuclei-templates/exposed-panels/spiderfoot.yaml", - "interactsh_disable": True, - } - } - } - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "SpiderFoot

support@spiderfoot.net

"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "FINDING" and "SpiderFoot" in e.data["description"] for e in events) - - -class Url_manipulation(HttpxMockHelper): - body = """ - - the title - -

Hello null!

'; - - - """ - - body_match = """ - - the title - -

Hello AAAAAAAAAAAAAA!

'; - - - """ - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"query_string": f"{self.module.rand_string}=.xml".encode()} - respond_args = {"response_data": self.body_match} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - respond_args = {"response_data": self.body} - self.set_expect_requests(respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "FINDING" - and e.data["description"] - == f"Url Manipulation: [body] Sig: [Modified URL: http://127.0.0.1:8888/?{self.module.rand_string}=.xml]" - for e in events - ) - - -class Naabu(HttpxMockHelper): - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "OPEN_TCP_PORT" for e in events) - - -class Social(HttpxMockHelper): - additional_modules = ["httpx", "excavate"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": ''} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "SOCIAL" and e.data["platform"] == "discord" for e in events) - - -class Hunt(HttpxMockHelper): - additional_modules = ["httpx"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": 'ping'} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any( - e.type == "FINDING" and e.data["description"] == "Found potential INSECURE CRYPTOGRAPHY parameter [cipher]" - for e in events - ) - - -class Bypass403(HttpxMockHelper): - additional_modules = ["httpx"] - - targets = ["http://127.0.0.1:8888/test"] - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 403 - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/test..;/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "FINDING" for e in events) - - -class Bypass403_aspnetcookieless(HttpxMockHelper): - additional_modules = ["httpx"] - - targets = ["http://127.0.0.1:8888/admin.aspx"] - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 403 - - def mock_args(self): - expect_args = {"method": "GET", "uri": re.compile(r"\/\([sS]\(\w+\)\)\/.+\.aspx")} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "FINDING" for e in events) - - -class Bypass403_waf(HttpxMockHelper): - additional_modules = ["httpx"] - - targets = ["http://127.0.0.1:8888/test"] - - def setup(self, scan): - self.bbot_httpserver.no_handler_status_code = 403 - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/test..;/"} - respond_args = {"response_data": "The requested URL was rejected"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert not any(e.type == "FINDING" for e in events) - - -class Speculate_subdirectories(HttpxMockHelper): - additional_modules = ["httpx"] - targets = ["http://127.0.0.1:8888/subdir1/subdir2/"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/subdir1/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/subdir1/subdir2/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - assert any(e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/subdir1/" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py index e69de29bb2..2d9cb1a7ae 100644 --- a/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py @@ -0,0 +1,6 @@ +from .base import ModuleTestBase + + +class TestGeneric_SSRF(ModuleTestBase): + # PAUL TODO + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_gowitness.py b/bbot/test/test_step_1/module_tests/test_module_gowitness.py index 278eb21a3f..9b3cc08082 100644 --- a/bbot/test/test_step_1/module_tests/test_module_gowitness.py +++ b/bbot/test/test_step_1/module_tests/test_module_gowitness.py @@ -22,7 +22,6 @@ def setup_after_prep(self, module_test): } module_test.set_expect_requests(respond_args=respond_args) - def check(self, module_test, events): screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots" screenshots = list(screenshots_path.glob("*.png")) diff --git a/bbot/test/test_step_1/module_tests/test_module_hackertarget.py b/bbot/test/test_step_1/module_tests/test_module_hackertarget.py index e69de29bb2..d32faa7bb5 100644 --- a/bbot/test/test_step_1/module_tests/test_module_hackertarget.py +++ b/bbot/test/test_step_1/module_tests/test_module_hackertarget.py @@ -0,0 +1,13 @@ +from .base import ModuleTestBase + + +class TestHackertarget(ModuleTestBase): + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.hackertarget.com/hostsearch/?q=blacklanternsecurity.com", + text="asdf.blacklanternsecurity.com\nzzzz.blacklanternsecurity.com", + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_host_header.py b/bbot/test/test_step_1/module_tests/test_module_host_header.py index e69de29bb2..75f24378e7 100644 --- a/bbot/test/test_step_1/module_tests/test_module_host_header.py +++ b/bbot/test/test_step_1/module_tests/test_module_host_header.py @@ -0,0 +1,6 @@ +from .base import ModuleTestBase + + +class TestHost_Header(ModuleTestBase): + # PAUL TODO + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_http.py b/bbot/test/test_step_1/module_tests/test_module_http.py index e69de29bb2..d1dd57daa6 100644 --- a/bbot/test/test_step_1/module_tests/test_module_http.py +++ b/bbot/test/test_step_1/module_tests/test_module_http.py @@ -0,0 +1,24 @@ +from .base import ModuleTestBase + + +class TestHTTP(ModuleTestBase): + downstream_url = "https://blacklanternsecurity.fakedomain:1234/events" + config_overrides = { + "output_modules": { + "http": { + "url": downstream_url, + "method": "PUT", + "bearer": "auth_token", + "username": "bbot_user", + "password": "bbot_password", + } + } + } + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + method="PUT", headers={"Authorization": "bearer auth_token"}, url=self.downstream_url + ) + + def check(self, module_test, events): + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_httpx.py b/bbot/test/test_step_1/module_tests/test_module_httpx.py index e69de29bb2..a737b2d762 100644 --- a/bbot/test/test_step_1/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_1/module_tests/test_module_httpx.py @@ -0,0 +1,29 @@ +import json +from .base import ModuleTestBase + + +class TestHTTPX(ModuleTestBase): + targets = ["http://127.0.0.1:8888/url", "127.0.0.1:8888"] + + def setup_after_prep(self, module_test): + request_args = dict(uri="/", headers={"test": "header"}) + respond_args = dict(response_data=json.dumps({"open": "port"})) + module_test.set_expect_requests(request_args, respond_args) + request_args = dict(uri="/url", headers={"test": "header"}) + respond_args = dict(response_data=json.dumps({"url": "url"})) + module_test.set_expect_requests(request_args, respond_args) + + def check(self, module_test, events): + url = False + open_port = False + for e in events: + if e.type == "HTTP_RESPONSE": + j = json.loads(e.data["body"]) + if e.data["path"] == "/": + if j.get("open", "") == "port": + open_port = True + elif e.data["path"] == "/url": + if j.get("url", "") == "url": + url = True + assert url, "Failed to visit target URL" + assert open_port, "Failed to visit target OPEN_TCP_PORT" diff --git a/bbot/test/test_step_1/module_tests/test_module_human.py b/bbot/test/test_step_1/module_tests/test_module_human.py index e69de29bb2..8bf252a002 100644 --- a/bbot/test/test_step_1/module_tests/test_module_human.py +++ b/bbot/test/test_step_1/module_tests/test_module_human.py @@ -0,0 +1,8 @@ +from .base import ModuleTestBase + + +class TestHuman(ModuleTestBase): + def check(self, module_test, events): + txt_file = module_test.scan.home / "output.txt" + with open(txt_file) as f: + assert f.read().startswith("[SCAN]") diff --git a/bbot/test/test_step_1/module_tests/test_module_hunt.py b/bbot/test/test_step_1/module_tests/test_module_hunt.py index e69de29bb2..d5809fa1e0 100644 --- a/bbot/test/test_step_1/module_tests/test_module_hunt.py +++ b/bbot/test/test_step_1/module_tests/test_module_hunt.py @@ -0,0 +1,17 @@ +from .base import ModuleTestBase + + +class TestHunt(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "hunt"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": 'ping'} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "FINDING" and e.data["description"] == "Found potential INSECURE CRYPTOGRAPHY parameter [cipher]" + for e in events + ) diff --git a/bbot/test/test_step_1/module_tests/test_module_hunterio.py b/bbot/test/test_step_1/module_tests/test_module_hunterio.py index e69de29bb2..ff09ed0d8b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_hunterio.py +++ b/bbot/test/test_step_1/module_tests/test_module_hunterio.py @@ -0,0 +1,96 @@ +from .base import ModuleTestBase + + +class TestHunterio(ModuleTestBase): + config_overrides = {"modules": {"hunterio": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.hunter.io/v2/account?api_key=asdf", + json={ + "data": { + "first_name": "jon", + "last_name": "snow", + "email": "jon@blacklanternsecurity.notreal", + "plan_name": "Starter", + "plan_level": 1, + "reset_date": "1917-05-23", + "team_id": 1234, + "calls": { + "_deprecation_notice": "Sums the searches and the verifications, giving an unprecise look of the available requests", + "used": 999, + "available": 2000, + }, + "requests": { + "searches": {"used": 998, "available": 1000}, + "verifications": {"used": 0, "available": 1000}, + }, + } + }, + ) + module_test.httpx_mock.add_response( + url="https://api.hunter.io/v2/domain-search?domain=blacklanternsecurity.com&api_key=asdf&limit=100&offset=0", + json={ + "data": { + "domain": "blacklanternsecurity.com", + "disposable": False, + "webmail": False, + "accept_all": False, + "pattern": "{first}", + "organization": "Black Lantern Security", + "description": None, + "twitter": None, + "facebook": None, + "linkedin": "https://linkedin.com/company/black-lantern-security", + "instagram": None, + "youtube": None, + "technologies": ["jekyll", "nginx"], + "country": "US", + "state": "CA", + "city": "Night City", + "postal_code": "12345", + "street": "123 Any St", + "emails": [ + { + "value": "asdf@blacklanternsecurity.com", + "type": "generic", + "confidence": 77, + "sources": [ + { + "domain": "blacklanternsecurity.com", + "uri": "http://blacklanternsecurity.com", + "extracted_on": "2021-06-09", + "last_seen_on": "2023-03-21", + "still_on_page": True, + } + ], + "first_name": None, + "last_name": None, + "position": None, + "seniority": None, + "department": "support", + "linkedin": None, + "twitter": None, + "phone_number": None, + "verification": {"date": None, "status": None}, + } + ], + "linked_domains": [], + }, + "meta": { + "results": 1, + "limit": 100, + "offset": 0, + "params": { + "domain": "blacklanternsecurity.com", + "company": None, + "type": None, + "seniority": None, + "department": None, + }, + }, + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf@blacklanternsecurity.com" for e in events), "Failed to detect email" diff --git a/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py b/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py index e69de29bb2..338d4e3b3c 100644 --- a/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py +++ b/bbot/test/test_step_1/module_tests/test_module_iis_shortnames.py @@ -0,0 +1,56 @@ +import re + +from .base import ModuleTestBase + + +class TestIIS_Shortnames(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "iis_shortnames"] + config_overrides = {"modules": {"iis_shortnames": {"detect_only": False}}} + + def setup_after_prep(self, module_test): + module_test.httpserver.no_handler_status_code = 404 + + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive", "status": 200} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/*~1*/a.aspx"} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/B\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/BL\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/BLS\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/BLSH\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/BLSHA\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": re.compile(r"\/BLSHAX\*~1\*.*$")} + respond_args = {"response_data": "", "status": 400} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + vulnerabilityEmitted = False + url_hintEmitted = False + for e in events: + if e.type == "VULNERABILITY": + vulnerabilityEmitted = True + if e.type == "URL_HINT" and e.data == "http://127.0.0.1:8888/BLSHAX~1": + url_hintEmitted = True + + assert vulnerabilityEmitted + assert url_hintEmitted diff --git a/bbot/test/test_step_1/test_modules_full.py b/bbot/test/test_step_1/test_modules_full.py deleted file mode 100644 index 5c6ec38651..0000000000 --- a/bbot/test/test_step_1/test_modules_full.py +++ /dev/null @@ -1,234 +0,0 @@ -import logging - -from ..bbot_fixtures import * # noqa: F401 -from ..modules_test_classes import * - -log = logging.getLogger(f"bbot.test") - - -@pytest.mark.asyncio -async def test_httpx(request): - x = Httpx(request) - await x.run() - - -@pytest.mark.asyncio -async def test_gowitness(request): - x = Gowitness(request) - await x.run() - - -@pytest.mark.asyncio -async def test_excavate(request): - x = Excavate(request) - await x.run() - - -@pytest.mark.asyncio -async def test_excavate_relativelinks(request): - x = Excavate_relativelinks(request, module_name="excavate") - await x.run() - - -@pytest.mark.asyncio -async def test_subdomain_hijack(request): - x = Subdomain_Hijack(request) - await x.run() - - -@pytest.mark.asyncio -async def test_fingerprintx(request): - x = Fingerprintx(request) - await x.run() - - -@pytest.mark.asyncio -async def test_otx(request): - x = Otx(request) - await x.run() - - -@pytest.mark.asyncio -async def test_anubisdb(request): - x = Anubisdb(request) - await x.run() - - -@pytest.mark.asyncio -async def test_secretsdb(request): - x = SecretsDB(request) - await x.run() - - -@pytest.mark.asyncio -async def test_badsecrets(request): - x = Badsecrets(request) - await x.run() - - -@pytest.mark.asyncio -async def test_telerik(request): - x = Telerik(request) - await x.run() - - -@pytest.mark.asyncio -async def test_paramminer_headers(request): - x = Paramminer_headers(request) - await x.run() - - -@pytest.mark.asyncio -async def test_paramminer_getparams(request): - x = Paramminer_getparams(request) - await x.run() - - -@pytest.mark.asyncio -async def test_paramminer_cookies(request): - x = Paramminer_cookies(request) - await x.run() - - -@pytest.mark.asyncio -async def test_leakix(request): - x = LeakIX(request) - await x.run() - - -@pytest.mark.asyncio -async def test_massdns(request): - x = Massdns(request) - await x.run() - - -@pytest.mark.asyncio -async def test_masscan(request): - x = Masscan(request) - await x.run() - - -@pytest.mark.asyncio -async def test_robots(request): - x = Robots(request) - await x.run() - - -@pytest.mark.asyncio -async def test_buckets(request): - x = Buckets(request, module_name="excavate") - await x.run() - - -@pytest.mark.asyncio -async def test_asn(request): - x = ASN(request) - await x.run() - - -@pytest.mark.asyncio -async def test_wafw00f(request): - x = Wafw00f(request) - await x.run() - - -@pytest.mark.asyncio -async def test_ffuf(request): - x = Ffuf(request) - await x.run() - - -@pytest.mark.asyncio -async def test_ffuf_extensions(request): - x = Ffuf_extensions(request, module_name="ffuf") - await x.run() - - -@pytest.mark.asyncio -async def test_vhost(request): - x = Vhost(request) - await x.run() - - -@pytest.mark.asyncio -async def test_ffuf_shortnames(request): - x = Ffuf_shortnames(request) - await x.run() - - -@pytest.mark.asyncio -async def test_iis_shortnames(request): - x = Iis_shortnames(request) - await x.run() - - -@pytest.mark.asyncio -async def test_nuclei_manual(request): - x = Nuclei_manual(request, module_name="nuclei") - await x.run() - - -@pytest.mark.asyncio -async def test_bypass403(request): - x = Bypass403(request) - await x.run() - - -@pytest.mark.asyncio -async def test_bypass403_waf(request): - x = Bypass403_waf(request, module_name="bypass403") - await x.run() - - -@pytest.mark.asyncio -async def test_bypass403_aspnetcookieless(request): - x = Bypass403_aspnetcookieless(request, module_name="bypass403") - await x.run() - - -@pytest.mark.asyncio -async def test_nuclei_technology(request, caplog): - x = Nuclei_technology(request, caplog, module_name="nuclei") - await x.run() - - -@pytest.mark.asyncio -async def test_nuclei_severe(request): - x = Nuclei_severe(request, module_name="nuclei") - await x.run() - - -@pytest.mark.asyncio -async def test_nuclei_budget(request): - x = Nuclei_budget(request, module_name="nuclei") - await x.run() - - -@pytest.mark.asyncio -async def test_url_manipulation(request): - x = Url_manipulation(request) - await x.run() - - -@pytest.mark.asyncio -async def test_naabu(request): - x = Naabu(request) - await x.run() - - -@pytest.mark.asyncio -async def test_hunt(request): - x = Hunt(request) - await x.run() - - -@pytest.mark.asyncio -async def test_speculate_subdirectories(request): - x = Speculate_subdirectories(request, module_name="speculate") - await x.run() - - -@pytest.mark.asyncio -async def test_social(request): - x = Social(request) - await x.run() diff --git a/bbot/test/test_step_2/test_scope.py b/bbot/test/test_step_2/test_scope.py index 55838d3458..cc39902ee9 100644 --- a/bbot/test/test_step_2/test_scope.py +++ b/bbot/test/test_step_2/test_scope.py @@ -1,48 +1,22 @@ from ..bbot_fixtures import * # noqa: F401 -from ..modules_test_classes import HttpxMockHelper +from ..test_step_1.module_tests.base import ModuleTestBase -class Scope_test_blacklist(HttpxMockHelper): - additional_modules = ["httpx"] +class Scope_test_blacklist(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx"] blacklist = ["127.0.0.1"] - def mock_args(self): + def setup_after_prep(self, module_test): expect_args = {"method": "GET", "uri": "/"} respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - def check_events(self, events): - for e in events: - if e.type == "URL": - return False - return True + def check(self, module_test, events): + assert not any(e.type == "URL" for e in events) -class Scope_test_whitelist(HttpxMockHelper): - additional_modules = ["httpx"] - +class Scope_test_whitelist(Scope_test_blacklist): + blacklist = [] whitelist = ["255.255.255.255"] - - def mock_args(self): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "alive"} - self.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check_events(self, events): - for e in events: - if e.type == "URL": - return False - return True - - -@pytest.mark.asyncio -async def test_scope_blacklist(bbot_config, bbot_scanner, bbot_httpserver): - x = Scope_test_blacklist(bbot_config, bbot_scanner, bbot_httpserver, module_name="httpx") - await x.run() - - -@pytest.mark.asyncio -async def test_scope_whitelist(bbot_config, bbot_scanner, bbot_httpserver): - x = Scope_test_whitelist(bbot_config, bbot_scanner, bbot_httpserver, module_name="httpx") - await x.run() diff --git a/bbot/test/test_step_2/test_web.py b/bbot/test/test_step_2/test_web.py index 92fcfa1afe..b0c76bf4fc 100644 --- a/bbot/test/test_step_2/test_web.py +++ b/bbot/test/test_step_2/test_web.py @@ -104,9 +104,14 @@ async def async_callback(data): log.debug(f"interactsh poll: {data}") interactsh_domain = await interactsh_client.register(callback=async_callback) + url = f"https://{interactsh_domain}/bbot_interactsh_test" + response = await scan1.helpers.request(url) + assert response.status_code == 200 + await asyncio.sleep(10) assert any(interactsh_domain.endswith(f"{s}") for s in server_list) data_list = await interactsh_client.poll() assert isinstance(data_list, list) + assert any("bbot_interactsh_test" in d.get("raw-request", "") for d in data_list) assert await interactsh_client.deregister() is None From b8b15345e03154176fe1b4663d3ca0e83ffd6785 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 21 May 2023 13:27:23 -0400 Subject: [PATCH 032/387] more module tests --- bbot/cli.py | 2 + bbot/core/errors.py | 2 +- bbot/core/helpers/dns.py | 6 + bbot/core/helpers/misc.py | 7 +- bbot/modules/dnszonetransfer.py | 6 +- bbot/modules/ipstack.py | 6 +- bbot/modules/leakix.py | 25 +++- bbot/modules/ntlm.py | 25 ++-- bbot/modules/output/neo4j.py | 14 +- bbot/modules/telerik.py | 6 +- bbot/scanner/scanner.py | 11 +- bbot/test/test_step_1/module_tests/base.py | 26 +++- .../test_module_dnszonetransfer.py | 17 +-- .../module_tests/test_module_fullhunt.py | 4 + .../module_tests/test_module_ipneighbor.py | 20 +++ .../module_tests/test_module_ipstack.py | 70 ++++++++++ .../module_tests/test_module_json.py | 14 ++ .../module_tests/test_module_leakix.py | 26 ++++ .../module_tests/test_module_masscan.py | 49 +++++++ .../module_tests/test_module_massdns.py | 16 +++ .../module_tests/test_module_naabu.py | 11 ++ .../module_tests/test_module_neo4j.py | 16 +++ .../module_tests/test_module_ntlm.py | 23 ++++ .../module_tests/test_module_nuclei.py | 125 ++++++++++++++++++ .../module_tests/test_module_otx.py | 27 ++++ .../test_module_paramminer_cookies.py | 46 +++++++ .../test_module_paramminer_getparams.py | 45 +++++++ .../test_module_paramminer_headers.py | 49 +++++++ 28 files changed, 641 insertions(+), 53 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index bc292f7c75..32e2a8c33c 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -327,6 +327,8 @@ def main(): global scan_name try: asyncio.run(_main()) + except asyncio.CancelledError: + pass except KeyboardInterrupt: msg = "Interrupted" if scan_name: diff --git a/bbot/core/errors.py b/bbot/core/errors.py index 24c94d4343..df134f66c2 100644 --- a/bbot/core/errors.py +++ b/bbot/core/errors.py @@ -1,4 +1,4 @@ -from httpx import RequestError # noqa +from httpx import RequestError, ReadTimeout # noqa class BBOTError(Exception): diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 3c10bcc48d..8135151ac2 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -432,6 +432,9 @@ async def is_wildcard(self, query, ips=None, rdtype=None): """ result = {} + if is_ip(query): + return result + # skip check if the query's parent domain is excluded in the config for d in self.wildcard_ignore: if self.parent_helper.host_in_host(query, d): @@ -518,6 +521,9 @@ async def is_wildcard_domain(self, domain, log_info=False): wildcard_domain_results = {} domain = self._clean_dns_record(domain) + if is_ip(domain): + return {} + # skip check if the query's parent domain is excluded in the config for d in self.wildcard_ignore: if self.parent_helper.host_in_host(domain, d): diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index ed8c414e37..9f2fa15db8 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1066,5 +1066,10 @@ def get_traceback_details(e): async def cancel_tasks(tasks): for task in tasks: task.cancel() - with suppress(asyncio.CancelledError): + try: await task + except asyncio.CancelledError: + pass + except Exception as e: + log.debug(e) + log.trace(traceback.format_exc()) diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index 52a9b9634d..5d959bcfe9 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -1,6 +1,5 @@ import dns.zone import dns.query -from functools import partial from bbot.modules.base import BaseModule @@ -37,8 +36,9 @@ async def handle_event(self, event): break try: self.debug(f"Attempting zone transfer against {nameserver} for domain {domain}") - xfr_fn = partial(dns.query.xfr, timeout=self.timeout, lifetime=self.timeout) - xfr_answer = await self.scan.run_in_executor(xfr_fn, nameserver, domain) + xfr_answer = await self.scan.run_in_executor( + dns.query.xfr, nameserver, domain, timeout=self.timeout, lifetime=self.timeout + ) zone = dns.zone.from_xfr(xfr_answer) except Exception as e: self.verbose(f"Error retrieving zone: {e}") diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index a5636ed62f..a1d01acc7c 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -19,8 +19,12 @@ class Ipstack(shodan_dns): base_url = "http://api.ipstack.com/" + async def filter_event(self, event): + return True + async def ping(self): - r = await self.request_with_fail_count(f"{self.base_url}/check?access_key={self.api_key}") + url = f"{self.base_url}/check?access_key={self.api_key}" + r = await self.request_with_fail_count(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index d494683afc..6622242e83 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -1,17 +1,38 @@ from .crobat import crobat +from .shodan_dns import shodan_dns -class leakix(crobat): +class leakix(shodan_dns): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive", "safe"] + options = {"api_key": ""} + # NOTE: API key is not required (but having one will get you more results) + options_desc = {"api_key": "LeakIX API Key"} meta = {"description": "Query leakix.net for subdomains"} base_url = "https://leakix.net" + async def setup(self): + ret = await crobat.setup(self) + self.headers = {"Accept": "application/json"} + self.api_key = self.config.get("api_key", "") + if self.api_key: + self.headers["api-key"] = self.api_key + return await self.require_api_key() + return ret + + async def ping(self): + url = f"{self.base_url}/host/1.2.3.4.5" + r = await self.helpers.request(url, headers=self.headers) + resp_content = getattr(r, "text", "") + assert getattr(r, "status_code", 0) != 401, resp_content + async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" - return await self.request_with_fail_count(url, headers={"Accept": "application/json"}) + response = await self.request_with_fail_count(url, headers=self.headers) + self.hugewarning(response.json()) + return response def parse_results(self, r, query=None): json = r.json() diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 6ef511560c..a5d2fed372 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -1,5 +1,5 @@ from bbot.modules.base import BaseModule -from bbot.core.errors import NTLMError, RequestError +from bbot.core.errors import NTLMError, RequestError, ReadTimeout ntlm_discovery_endpoints = [ "", @@ -77,19 +77,21 @@ async def setup(self): async def handle_event(self, event): found_hash = hash(f"{event.host}:{event.port}") if found_hash not in self.found: - result_FQDN, request_url = self.handle_url(event) - if result_FQDN and request_url: + result, request_url = await self.handle_url(event) + if result and request_url: self.found.add(found_hash) self.emit_event( { "host": str(event.host), "url": request_url, - "description": f"NTLM AUTH: {result_FQDN}", + "description": f"NTLM AUTH: {result}", }, "FINDING", source=event, ) - self.emit_event(result_FQDN, "DNS_NAME", source=event) + fqdn = result.get("FQDN", "") + if fqdn: + self.emit_event(fqdn, "DNS_NAME", source=event) async def filter_event(self, event): if self.try_all: @@ -127,12 +129,13 @@ async def handle_url(self, event): try: result, url = await task if result: - self.helpers.cancel_tasks(tasks) - return str(result["FQDN"]), url - except RequestError as e: - self.warning(str(e)) + await self.helpers.cancel_tasks(tasks) + return result, url + except (RequestError, ReadTimeout) as e: + if str(e): + self.warning(str(e)) # cancel all the tasks if there's an error - self.helpers.cancel_tasks(tasks) + await self.helpers.cancel_tasks(tasks) break return None, None @@ -152,3 +155,5 @@ async def check_ntlm(self, test_url): return ntlm_resp_decoded, test_url except NTLMError as e: self.verbose(str(e)) + return None, test_url + return None, test_url diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index ea7e3ff720..477bdd373b 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -1,5 +1,4 @@ from bbot.db.neo4j import Neo4j - from bbot.modules.output.base import BaseOutputModule @@ -21,19 +20,20 @@ class neo4j(BaseOutputModule): async def setup(self): try: - self.neo4j = Neo4j( + self.neo4j = await self.scan.run_in_executor( + Neo4j, uri=self.config.get("uri", self.options["uri"]), username=self.config.get("username", self.options["username"]), password=self.config.get("password", self.options["password"]), ) - self.neo4j.insert_event(self.scan.root_event) + await self.scan.run_in_executor(self.neo4j.insert_event, self.scan.root_event) except Exception as e: self.warning(f"Error setting up Neo4j: {e}") return False return True - def handle_event(self, event): - self.neo4j.insert_event(event) + async def handle_event(self, event): + await self.scan.run_in_executor(self.neo4j.insert_event, event) - def handle_batch(self, *events): - self.neo4j.insert_events(events) + async def handle_batch(self, *events): + await self.scan.run_in_executor(self.neo4j.insert_events, events) diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 5166e45b73..1fa5cc6425 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -235,11 +235,11 @@ async def handle_event(self, event): if fail_count < 2: continue self.debug(f"Cancelling run against {event.data} due to failed request") - self.helpers.cancel_tasks(tasks) + await self.helpers.cancel_tasks(tasks) break else: if "Cannot deserialize dialog parameters" in result.text: - self.helpers.cancel_tasks(tasks) + await self.helpers.cancel_tasks(tasks) self.debug(f"Detected Telerik UI instance ({dh})") description = f"Telerik DialogHandler detected" self.emit_event( @@ -250,7 +250,7 @@ async def handle_event(self, event): # Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard break - self.helpers.cancel_tasks(tasks) + await self.helpers.cancel_tasks(tasks) spellcheckhandler = "Telerik.Web.UI.SpellCheckHandler.axd" result, _ = await self.test_detector(event.data, spellcheckhandler) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c2459ad1cd..f55a5c0d60 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -8,6 +8,7 @@ from sys import exc_info from pathlib import Path from datetime import datetime +from functools import partial from omegaconf import OmegaConf from collections import OrderedDict @@ -381,10 +382,7 @@ async def cancel_tasks(self): tasks.append(self.ticker_task) # manager worker loops tasks += self.manager_worker_loop_tasks - for t in tasks: - t.cancel() - with contextlib.suppress(asyncio.CancelledError): - await t + await self.helpers.cancel_tasks(tasks) async def report(self): for mod in self.modules.values(): @@ -675,8 +673,9 @@ async def acatch(self, context="scan", finally_callback=None): except BaseException as e: self._handle_exception(e, context=context) - def run_in_executor(self, *args, **kwargs): - return self._loop.run_in_executor(None, *args, **kwargs) + def run_in_executor(self, callback, *args, **kwargs): + callback = partial(callback, **kwargs) + return self._loop.run_in_executor(None, callback, *args) def _handle_exception(self, e, context="scan", finally_callback=None): if callable(context): diff --git a/bbot/test/test_step_1/module_tests/base.py b/bbot/test/test_step_1/module_tests/base.py index 2f5f935591..41294b2c6c 100644 --- a/bbot/test/test_step_1/module_tests/base.py +++ b/bbot/test/test_step_1/module_tests/base.py @@ -1,8 +1,9 @@ import pytest -import pytest_asyncio import logging +import pytest_asyncio from abc import abstractmethod from omegaconf import OmegaConf +from types import SimpleNamespace from bbot.scanner import Scanner from bbot.modules import module_loader @@ -25,6 +26,19 @@ def tempwordlist(content): return filename +class MockRecord: + def __init__(self, record, rdtype): + self.rdtype = SimpleNamespace() + self.rdtype.name = rdtype + self.record = record + + def __str__(self): + return self.record + + def to_text(self): + return str(self) + + class TestClass: @pytest_asyncio.fixture async def my_fixture(self, bbot_httpserver): @@ -45,13 +59,14 @@ class ModuleTestBase: modules_overrides = [] class ModuleTest: - def __init__(self, module_test_base, httpx_mock, httpserver, monkeypatch): + def __init__(self, module_test_base, httpx_mock, httpserver, monkeypatch, request): self.name = module_test_base.name self.config = OmegaConf.merge(test_config, OmegaConf.create(module_test_base.config_overrides)) self.httpx_mock = httpx_mock self.httpserver = httpserver self.monkeypatch = monkeypatch + self.request_fixture = request # handle output, internal module types preloaded = module_loader.preloaded() @@ -90,9 +105,12 @@ def set_expect_requests(self, expect_args={}, respond_args={}): def module(self): return self.scan.modules[self.name] + def mock_record(self, *args, **kwargs): + return MockRecord(*args, **kwargs) + @pytest_asyncio.fixture - async def module_test(self, httpx_mock, bbot_httpserver, monkeypatch): - module_test = self.ModuleTest(self, httpx_mock, bbot_httpserver, monkeypatch) + async def module_test(self, httpx_mock, bbot_httpserver, monkeypatch, request): + module_test = self.ModuleTest(self, httpx_mock, bbot_httpserver, monkeypatch, request) self.setup_before_prep(module_test) await module_test.scan.prep() self.setup_after_prep(module_test) diff --git a/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py index f746ee717d..40f450a974 100644 --- a/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py +++ b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py @@ -1,7 +1,6 @@ import dns.zone import dns.query import dns.message -from types import SimpleNamespace from .base import ModuleTestBase @@ -13,23 +12,11 @@ class TestDNSZoneTransfer(ModuleTestBase): def setup_after_prep(self, module_test): old_resolve_fn = module_test.scan.helpers.dns._resolve_hostname - class MockRecord: - def __init__(self, record, rdtype): - self.rdtype = SimpleNamespace() - self.rdtype.name = rdtype - self.record = record - - def __str__(self): - return self.record - - def to_text(self): - return str(self) - async def _resolve_hostname(query, **kwargs): if query == "blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() == "NS": - return [MockRecord("ns01.blacklanternsecurity.fakedomain", "NS")], [] + return [module_test.mock_record("ns01.blacklanternsecurity.fakedomain", "NS")], [] if query == "ns01.blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() in "A": - return [MockRecord("127.0.0.1", "A")], [] + return [module_test.mock_record("127.0.0.1", "A")], [] return await old_resolve_fn(query, **kwargs) def from_xfr(*args, **kwargs): diff --git a/bbot/test/test_step_1/module_tests/test_module_fullhunt.py b/bbot/test/test_step_1/module_tests/test_module_fullhunt.py index bada14732a..1847ad40c7 100644 --- a/bbot/test/test_step_1/module_tests/test_module_fullhunt.py +++ b/bbot/test/test_step_1/module_tests/test_module_fullhunt.py @@ -2,9 +2,12 @@ class TestFullhunt(ModuleTestBase): + config_overrides = {"modules": {"fullhunt": {"api_key": "asdf"}}} + def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( url="https://fullhunt.io/api/v1/auth/status", + match_headers={"x-api-key": "asdf"}, json={ "message": "", "status": 200, @@ -25,6 +28,7 @@ def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://fullhunt.io/api/v1/domain/blacklanternsecurity.com/subdomains", + match_headers={"x-api-key": "asdf"}, json={ "domain": "blacklanternsecurity.com", "hosts": [ diff --git a/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py b/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py index e69de29bb2..6463c2cfaf 100644 --- a/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py +++ b/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py @@ -0,0 +1,20 @@ +from .base import ModuleTestBase + + +class TestIPNeighbor(ModuleTestBase): + targets = ["127.0.0.15", "www.bls.notreal"] + config_overrides = {"scope_report_distance": 1, "dns_resolution": True} + + def setup_after_prep(self, module_test): + old_resolve_fn = module_test.scan.helpers.dns.resolve + + async def resolve(query, **kwargs): + module_test.log.critical(f"{query}: {kwargs}") + if query == "127.0.0.3" and kwargs.get("type", "").upper() == "PTR": + return {"www.bls.notreal"} + return await old_resolve_fn(query, **kwargs) + + module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "resolve", resolve) + + def check(self, module_test, events): + assert any(e.data == "127.0.0.3" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_ipstack.py b/bbot/test/test_step_1/module_tests/test_module_ipstack.py index e69de29bb2..8e45c338c2 100644 --- a/bbot/test/test_step_1/module_tests/test_module_ipstack.py +++ b/bbot/test/test_step_1/module_tests/test_module_ipstack.py @@ -0,0 +1,70 @@ +from .base import ModuleTestBase + + +class TestIPStack(ModuleTestBase): + targets = ["8.8.8.8"] + config_overrides = {"modules": {"ipstack": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="http://api.ipstack.com//check?access_key=asdf", + json={ + "ip": "1.2.3.4", + "type": "ipv4", + "continent_code": "NA", + "continent_name": "North America", + "country_code": "US", + "country_name": "United States", + "region_code": "FL", + "region_name": "Florida", + "city": "Cape Canaveral", + "zip": "12345", + "latitude": 47.89263153076172, + "longitude": -97.04190063476562, + "location": { + "geoname_id": 5059429, + "capital": "Washington D.C.", + "languages": [{"code": "en", "name": "English", "native": "English"}], + "country_flag": "https://assets.ipstack.com/flags/us.svg", + "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8", + "country_flag_emoji_unicode": "U+1F1FA U+1F1F8", + "calling_code": "1", + "is_eu": False, + }, + }, + ) + module_test.httpx_mock.add_response( + url="http://api.ipstack.com//8.8.8.8?access_key=asdf", + json={ + "ip": "8.8.8.8", + "type": "ipv4", + "continent_code": "NA", + "continent_name": "North America", + "country_code": "US", + "country_name": "United States", + "region_code": "OH", + "region_name": "Ohio", + "city": "Glenmont", + "zip": "44628", + "latitude": 40.5369987487793, + "longitude": -82.12859344482422, + "location": { + "geoname_id": None, + "capital": "Washington D.C.", + "languages": [{"code": "en", "name": "English", "native": "English"}], + "country_flag": "https://assets.ipstack.com/flags/us.svg", + "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8", + "country_flag_emoji_unicode": "U+1F1FA U+1F1F8", + "calling_code": "1", + "is_eu": False, + }, + }, + ) + + def check(self, module_test, events): + assert any( + e.type == "GEOLOCATION" + and e.data + == "Ip: 8.8.8.8, Country: United States, City: Glenmont, Zip_code: 44628, Region: Ohio, Latitude: 40.5369987487793, Longitude: -82.12859344482422" + for e in events + ), "Failed to geolocate IP" diff --git a/bbot/test/test_step_1/module_tests/test_module_json.py b/bbot/test/test_step_1/module_tests/test_module_json.py index e69de29bb2..dd552742a5 100644 --- a/bbot/test/test_step_1/module_tests/test_module_json.py +++ b/bbot/test/test_step_1/module_tests/test_module_json.py @@ -0,0 +1,14 @@ +import json + +from .base import ModuleTestBase +from bbot.core.event.base import event_from_json + + +class TestJSON(ModuleTestBase): + def check(self, module_test, events): + txt_file = module_test.scan.home / "output.json" + lines = list(module_test.scan.helpers.read_file(txt_file)) + assert lines + e = event_from_json(json.loads(lines[0])) + assert e.type == "SCAN" + assert e.data == f"{module_test.scan.name} ({module_test.scan.id})" diff --git a/bbot/test/test_step_1/module_tests/test_module_leakix.py b/bbot/test/test_step_1/module_tests/test_module_leakix.py index e69de29bb2..9266660f45 100644 --- a/bbot/test/test_step_1/module_tests/test_module_leakix.py +++ b/bbot/test/test_step_1/module_tests/test_module_leakix.py @@ -0,0 +1,26 @@ +from .base import ModuleTestBase + + +class TestLeakIX(ModuleTestBase): + config_overrides = {"modules": {"leakix": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://leakix.net/host/1.2.3.4.5", + match_headers={"api-key": "asdf"}, + json={"title": "Not Found", "description": "Host not found"}, + ) + module_test.httpx_mock.add_response( + url=f"https://leakix.net/api/subdomains/blacklanternsecurity.com", + match_headers={"api-key": "asdf"}, + json=[ + { + "subdomain": "asdf.blacklanternsecurity.com", + "distinct_ips": 3, + "last_seen": "2023-04-02T09:38:30.02Z", + }, + ], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_masscan.py b/bbot/test/test_step_1/module_tests/test_module_masscan.py index e69de29bb2..2d4fd44631 100644 --- a/bbot/test/test_step_1/module_tests/test_module_masscan.py +++ b/bbot/test/test_step_1/module_tests/test_module_masscan.py @@ -0,0 +1,49 @@ +from .base import ModuleTestBase + + +class TestMasscan(ModuleTestBase): + targets = ["8.8.8.8/32"] + scan_name = "test_masscan" + config_overrides = {"modules": {"masscan": {"ports": "443", "wait": 1}}} + masscan_config = """seed = 17230484647655100360 +rate = 600 +shard = 1/1 + + +# TARGET SELECTION (IP, PORTS, EXCLUDES) +ports = +range = 9.8.7.6""" + + masscan_output = """[ +{ "ip": "8.8.8.8", "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] } +]""" + + def setup_after_prep(self, module_test): + self.masscan_run = False + + async def run_masscan(command, *args, **kwargs): + if "masscan" in command[:2]: + for l in self.masscan_output.splitlines(): + yield l + self.masscan_run = True + else: + async for l in self.scan.helpers.run_live(command, *args, **kwargs): + yield l + + module_test.scan.modules["masscan"].masscan_config = self.masscan_config + module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan) + + def check(self, module_test, events): + assert self.masscan_run == True + assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" + assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" + + +class TestMasscan1(TestMasscan): + modules_overrides = ["masscan"] + config_overrides = {"modules": {"masscan": {"ports": "443", "wait": 1, "use_cache": True}}} + + def check(self, module_test, events): + assert self.masscan_run == False + assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" + assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" diff --git a/bbot/test/test_step_1/module_tests/test_module_massdns.py b/bbot/test/test_step_1/module_tests/test_module_massdns.py index e69de29bb2..5d2fcdfb61 100644 --- a/bbot/test/test_step_1/module_tests/test_module_massdns.py +++ b/bbot/test/test_step_1/module_tests/test_module_massdns.py @@ -0,0 +1,16 @@ +from .base import ModuleTestBase, tempwordlist + + +class TestMassdns(ModuleTestBase): + subdomain_wordlist = tempwordlist(["www", "asdf"]) + config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist)}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt", + text="8.8.8.8\n8.8.4.4\n1.1.1.1", + ) + + def check(self, module_test, events): + assert any(e.data == "www.blacklanternsecurity.com" for e in events) + assert not any(e.data == "asdf.blacklanternsecurity.com" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_naabu.py b/bbot/test/test_step_1/module_tests/test_module_naabu.py index e69de29bb2..2f985d5a22 100644 --- a/bbot/test/test_step_1/module_tests/test_module_naabu.py +++ b/bbot/test/test_step_1/module_tests/test_module_naabu.py @@ -0,0 +1,11 @@ +from .base import ModuleTestBase + + +class TestNaabu(ModuleTestBase): + def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "OPEN_TCP_PORT" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_neo4j.py b/bbot/test/test_step_1/module_tests/test_module_neo4j.py index e69de29bb2..97cb754e0a 100644 --- a/bbot/test/test_step_1/module_tests/test_module_neo4j.py +++ b/bbot/test/test_step_1/module_tests/test_module_neo4j.py @@ -0,0 +1,16 @@ +from .base import ModuleTestBase + + +class TestNeo4j(ModuleTestBase): + def setup_before_prep(self, module_test): + class MockGraph: + def __init__(self, *args, **kwargs): + self.used = False + + def merge(self, *args, **kwargs): + self.used = True + + module_test.monkeypatch.setattr("py2neo.Graph", MockGraph) + + def check(self, module_test, events): + assert module_test.scan.modules["neo4j"].neo4j.graph.used == True diff --git a/bbot/test/test_step_1/module_tests/test_module_ntlm.py b/bbot/test/test_step_1/module_tests/test_module_ntlm.py index e69de29bb2..814e2578e7 100644 --- a/bbot/test/test_step_1/module_tests/test_module_ntlm.py +++ b/bbot/test/test_step_1/module_tests/test_module_ntlm.py @@ -0,0 +1,23 @@ +from .base import ModuleTestBase + + +class TestNTLM(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "ntlm"] + config_overrides = {"modules": {"ntlm": {"try_all": True}}} + + def setup_after_prep(self, module_test): + request_args = dict(uri="/", headers={"test": "header"}) + module_test.set_expect_requests(request_args, {}) + request_args = dict( + uri="/oab/", headers={"Authorization": "NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw=="} + ) + respond_args = dict( + headers={ + "WWW-Authenticate": "NTLM TlRMTVNTUAACAAAABgAGADgAAAAVgoni89aZT4Q0mH0AAAAAAAAAAHYAdgA+AAAABgGxHQAAAA9WAE4ATwACAAYAVgBOAE8AAQAKAEUAWABDADAAMQAEABIAdgBuAG8ALgBsAG8AYwBhAGwAAwAeAEUAWABDADAAMQAuAHYAbgBvAC4AbABvAGMAYQBsAAUAEgB2AG4AbwAuAGwAbwBjAGEAbAAHAAgAXxo0p/6L2QEAAAAA" + } + ) + module_test.set_expect_requests(request_args, respond_args) + + def check(self, module_test, events): + assert any(e.type == "FINDING" and "EXC01.vno.local" in e.data["description"] for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_nuclei.py b/bbot/test/test_step_1/module_tests/test_module_nuclei.py index e69de29bb2..b8eca8a3b5 100644 --- a/bbot/test/test_step_1/module_tests/test_module_nuclei.py +++ b/bbot/test/test_step_1/module_tests/test_module_nuclei.py @@ -0,0 +1,125 @@ +from .base import ModuleTestBase + + +class TestNucleiManual(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "nuclei"] + config_overrides = { + "web_spider_distance": 1, + "web_spider_depth": 1, + "modules": { + "nuclei": { + "version": "2.9.4", + "mode": "manual", + "concurrency": 2, + "ratelimit": 10, + "templates": "/tmp/.bbot_test/tools/nuclei-templates/http/miscellaneous/", + "interactsh_disable": True, + "directory_only": False, + } + }, + } + + test_html = """ + html> + + Index of /test + + +

Index of /test

+ + + + +
NameLast modifiedSize
Parent Directory  -
+
Apache/2.4.38 (Debian) Server at http://127.0.0.1:8888/testmultipleruns.html
+ +""" + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": self.test_html} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + expect_args = {"method": "GET", "uri": "/testmultipleruns.html"} + respond_args = {"response_data": "Copyright 1984"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + first_run_detect = False + second_run_detect = False + for e in events: + if e.type == "FINDING": + if "Directory listing enabled" in e.data["description"]: + first_run_detect = True + elif "Copyright" in e.data["description"]: + second_run_detect = True + assert first_run_detect + assert second_run_detect + + +class TestNucleiSevere(TestNucleiManual): + modules_overrides = ["httpx", "nuclei"] + config_overrides = { + "modules": { + "nuclei": { + "mode": "severe", + "concurrency": 1, + "templates": "/tmp/.bbot_test/tools/nuclei-templates/vulnerabilities/generic/generic-linux-lfi.yaml", + } + }, + "interactsh_disable": True, + } + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/etc/passwd"} + respond_args = {"response_data": "root:.*:0:0:"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "VULNERABILITY" and "Generic Linux - Local File Inclusion" in e.data["description"] + for e in events + ) + + +class TestNucleiTechnology(TestNucleiManual): + config_overrides = { + "interactsh_disable": True, + "modules": {"nuclei": {"mode": "technology", "concurrency": 2, "tags": "apache"}}, + } + + def setup_before_prep(self, module_test): + self.caplog = module_test.request_fixture.getfixturevalue("caplog") + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": "", + "headers": {"Server": "Apache/2.4.52 (Ubuntu)"}, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + if "Using Interactsh Server" in self.caplog.text: + return False + assert any(e.type == "FINDING" and "apache" in e.data["description"] for e in events) + + +class TestNucleiBudget(TestNucleiManual): + config_overrides = { + "modules": { + "nuclei": { + "mode": "budget", + "concurrency": 1, + "tags": "spiderfoot", + "templates": "/tmp/.bbot_test/tools/nuclei-templates/exposed-panels/spiderfoot.yaml", + "interactsh_disable": True, + } + } + } + + def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "SpiderFoot

support@spiderfoot.net

"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "FINDING" and "SpiderFoot" in e.data["description"] for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_otx.py b/bbot/test/test_step_1/module_tests/test_module_otx.py index e69de29bb2..7481e29056 100644 --- a/bbot/test/test_step_1/module_tests/test_module_otx.py +++ b/bbot/test/test_step_1/module_tests/test_module_otx.py @@ -0,0 +1,27 @@ +from .base import ModuleTestBase + + +class TestOTX(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://otx.alienvault.com/api/v1/indicators/domain/blacklanternsecurity.com/passive_dns", + json={ + "passive_dns": [ + { + "address": "2606:50c0:8000::153", + "first": "2021-10-28T20:23:08", + "last": "2022-08-24T18:29:49", + "hostname": "asdf.blacklanternsecurity.com", + "record_type": "AAAA", + "indicator_link": "/indicator/hostname/www.blacklanternsecurity.com", + "flag_url": "assets/images/flags/us.png", + "flag_title": "United States", + "asset_type": "hostname", + "asn": "AS54113 fastly", + } + ] + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py index e69de29bb2..c0ebbb3e04 100644 --- a/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py +++ b/bbot/test/test_step_1/module_tests/test_module_paramminer_cookies.py @@ -0,0 +1,46 @@ +from .test_module_paramminer_headers import * + + +class TestParamminer_Cookies(TestParamminer_Headers): + modules_overrides = ["httpx", "paramminer_cookies"] + config_overrides = {"modules": {"paramminer_cookies": {"wordlist": tempwordlist(["junkcookie", "admincookie"])}}} + + cookies_body = """ + + the title + +

Hello null!

'; + + + """ + + cookies_body_match = """ + + the title + +

Hello AAAAAAAAAAAAAA!

'; + + + """ + + def setup_after_prep(self, module_test): + module_test.scan.modules["paramminer_cookies"].rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + module_test.monkeypatch.setattr( + helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"} + ) + expect_args = dict(headers={"Cookie": "admincookie=AAAAAAAAAAAAAA"}) + respond_args = {"response_data": self.cookies_body_match} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + respond_args = {"response_data": self.cookies_body} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [admincookie] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Cookie: [junkcookie] Reasons: [body]" + for e in events + ) diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py index e69de29bb2..f6cbdebeac 100644 --- a/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py +++ b/bbot/test/test_step_1/module_tests/test_module_paramminer_getparams.py @@ -0,0 +1,45 @@ +from .test_module_paramminer_headers import * + + +class TestParamminer_Getparams(TestParamminer_Headers): + modules_overrides = ["httpx", "paramminer_getparams"] + config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist(["canary", "id"])}}} + getparam_body = """ + + the title + +

Hello null!

'; + + + """ + + getparam_body_match = """ + + the title + +

Hello AAAAAAAAAAAAAA!

'; + + + """ + + def setup_after_prep(self, module_test): + module_test.scan.modules["paramminer_getparams"].rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + module_test.monkeypatch.setattr( + helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"} + ) + expect_args = {"query_string": b"id=AAAAAAAAAAAAAA&AAAAAA=1"} + respond_args = {"response_data": self.getparam_body_match} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + respond_args = {"response_data": self.getparam_body} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [id] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Getparam: [canary] Reasons: [body]" + for e in events + ) diff --git a/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py index e69de29bb2..7fc2e63959 100644 --- a/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py +++ b/bbot/test/test_step_1/module_tests/test_module_paramminer_headers.py @@ -0,0 +1,49 @@ +from bbot.core.helpers import helper + +from .base import ModuleTestBase, tempwordlist + + +class TestParamminer_Headers(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "paramminer_headers"] + config_overrides = {"modules": {"paramminer_headers": {"wordlist": tempwordlist(["junkword1", "tracestate"])}}} + + headers_body = """ + + the title + +

Hello null!

'; + + + """ + + headers_body_match = """ + + the title + +

Hello AAAAAAAAAAAAAA!

'; + + + """ + + def setup_after_prep(self, module_test): + module_test.scan.modules["paramminer_headers"].rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + module_test.monkeypatch.setattr( + helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"} + ) + expect_args = dict(headers={"tracestate": "AAAAAAAAAAAAAA"}) + respond_args = {"response_data": self.headers_body_match} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + respond_args = {"response_data": self.headers_body} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [tracestate] Reasons: [body]" + for e in events + ) + assert not any( + e.type == "FINDING" and e.data["description"] == "[Paramminer] Header: [junkword1] Reasons: [body]" + for e in events + ) From c915b557e34f1a062074728ff2effa28ec0eb44f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 21 May 2023 16:49:23 -0400 Subject: [PATCH 033/387] tests for passivetotal, pgp --- bbot/core/event/helpers.py | 4 +-- bbot/core/helpers/dns.py | 8 ++--- bbot/core/helpers/misc.py | 11 ++++++ bbot/core/helpers/regexes.py | 3 +- bbot/core/helpers/validators.py | 2 +- bbot/modules/crobat.py | 7 ++-- bbot/modules/passivetotal.py | 12 +++---- bbot/modules/pgp.py | 14 ++++---- .../module_tests/test_module_passivetotal.py | 21 +++++++++++ .../module_tests/test_module_pgp.py | 35 +++++++++++++++++++ bbot/test/test_step_2/test_helpers.py | 6 ++++ 11 files changed, 100 insertions(+), 23 deletions(-) diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py index 5820183ec2..7213bc57b5 100644 --- a/bbot/core/event/helpers.py +++ b/bbot/core/event/helpers.py @@ -4,7 +4,7 @@ from bbot.core.errors import ValidationError from bbot.core.helpers import sha1, smart_decode, smart_decode_punycode -from bbot.core.helpers.regexes import event_type_regexes, event_id_regex, _hostname_regex +from bbot.core.helpers.regexes import event_type_regexes, event_id_regex, hostname_regex log = logging.getLogger("bbot.core.event.helpers") @@ -36,7 +36,7 @@ def get_event_type(data): return t # Assume DNS_NAME for basic words - if _hostname_regex.match(data): + if hostname_regex.match(data): return "DNS_NAME" raise ValidationError(f'Unable to autodetect event type from "{data}"') diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 8135151ac2..48bb444cc4 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -9,7 +9,7 @@ from .regexes import dns_name_regex from bbot.core.helpers.async_helpers import NamedLock from bbot.core.errors import ValidationError, DNSError -from .misc import is_ip, is_domain, domain_parents, parent_domain, rand_string, cloudcheck +from .misc import is_ip, is_domain, is_dns_name, domain_parents, parent_domain, rand_string, cloudcheck log = logging.getLogger("bbot.core.helpers.dns") @@ -432,8 +432,8 @@ async def is_wildcard(self, query, ips=None, rdtype=None): """ result = {} - if is_ip(query): - return result + if not is_dns_name(query): + return {} # skip check if the query's parent domain is excluded in the config for d in self.wildcard_ignore: @@ -521,7 +521,7 @@ async def is_wildcard_domain(self, domain, log_info=False): wildcard_domain_results = {} domain = self._clean_dns_record(domain) - if is_ip(domain): + if not is_dns_name(domain): return {} # skip check if the query's parent domain is excluded in the config diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 9f2fa15db8..ec4b9b20ec 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -214,6 +214,17 @@ def is_port(p): return p and p.isdigit() and 0 <= int(p) <= 65535 +def is_dns_name(d): + if is_ip(d): + return False + d = smart_decode(d) + if regexes.hostname_regex.match(d): + return True + if regexes.dns_name_regex.match(d): + return True + return False + + def is_ip(d, version=None): """ "192.168.1.1" --> True diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index c739ebb029..634257ca5f 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -22,7 +22,7 @@ _ipv6_regex = r"[A-F0-9:]*:[A-F0-9:]*:[A-F0-9:]*" ipv6_regex = re.compile(_ipv6_regex, re.I) _dns_name_regex = r"(?:(?:[\w-]+)\.)+(?:[^\W_0-9]{2,20})" -_hostname_regex = re.compile(r"^[\w-]+$") +_hostname_regex = r"^[\w-]+$" _email_regex = r"(?:[^\W_][\w\-\.\+]{,100})@(?:\w[\w\-\._]{,100})\.(?:[^\W_0-9]{2,8})" email_regex = re.compile(_email_regex, re.I) _ptr_regex = r"(?:[0-9]{1,3}[-_\.]){3}[0-9]{1,3}" @@ -61,3 +61,4 @@ event_id_regex = re.compile(r"[0-9a-f]{40}:[A-Z0-9_]+") dns_name_regex = re.compile(_dns_name_regex, re.I) scan_name_regex = re.compile(r"[a-z]{3,20}_[a-z]{3,20}") +hostname_regex = re.compile(_hostname_regex, re.I) diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py index a1945802a5..5082a2dd31 100644 --- a/bbot/core/helpers/validators.py +++ b/bbot/core/helpers/validators.py @@ -62,7 +62,7 @@ def validate_host(host): for r in regexes.event_type_regexes["DNS_NAME"]: if r.match(host): return host - if regexes._hostname_regex.match(host): + if regexes.hostname_regex.match(host): return host assert False, f'Invalid hostname: "{host}"' diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py index da4191911f..1db822eb8d 100644 --- a/bbot/modules/crobat.py +++ b/bbot/modules/crobat.py @@ -29,9 +29,10 @@ async def setup(self): return True async def _is_wildcard(self, query): - for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query)).items(): - if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")): - return True + if self.helpers.is_dns_name(query): + for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query)).items(): + if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")): + return True return False async def filter_event(self, event): diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index 27220bca35..83ebd1d7ae 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -11,15 +11,15 @@ class passivetotal(shodan_dns): base_url = "https://api.passivetotal.org/v2" - def setup(self): + async def setup(self): self.username = self.config.get("username", "") self.api_key = self.config.get("api_key", "") self.auth = (self.username, self.api_key) - return super().setup() + return await super().setup() - def ping(self): + async def ping(self): url = f"{self.base_url}/account/quota" - j = self.request_with_fail_count(url, auth=self.auth).json() + j = (await self.request_with_fail_count(url, auth=self.auth)).json() limit = j["user"]["limits"]["search_api"] used = j["user"]["counts"]["search_api"] assert used < limit, "No quota remaining" @@ -28,9 +28,9 @@ def abort_if(self, event): # RiskIQ is famous for their junk data return super().abort_if(event) or "unresolved" in event.tags - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}" - return self.request_with_fail_count(url, auth=self.auth) + return await self.request_with_fail_count(url, auth=self.auth) def parse_results(self, r, query): for subdomain in r.json().get("subdomains", []): diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index fe967cbe22..ce7098e27d 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -14,20 +14,22 @@ class pgp(crobat): } options_desc = {"search_urls": "PGP key servers to search"} - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - results = self.query(query) + results = await self.query(query) if results: for hostname in results: if not hostname == event: self.emit_event(hostname, "EMAIL_ADDRESS", event, abort_if=self.abort_if) - def query(self, query): + async def query(self, query): + results = set() for url in self.config.get("search_urls", []): url = url.replace("", self.helpers.quote(query)) - response = self.helpers.request(url) + response = await self.helpers.request(url) if response is not None: for email in self.helpers.extract_emails(response.text): email = email.lower() - if email.lower().endswith(query): - yield email + if email.endswith(query): + results.add(email) + return results diff --git a/bbot/test/test_step_1/module_tests/test_module_passivetotal.py b/bbot/test/test_step_1/module_tests/test_module_passivetotal.py index e69de29bb2..5d2835979b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_passivetotal.py +++ b/bbot/test/test_step_1/module_tests/test_module_passivetotal.py @@ -0,0 +1,21 @@ +from .base import ModuleTestBase + + +class TestPassiveTotal(ModuleTestBase): + config_overrides = {"modules": {"passivetotal": {"username": "jon@bls.fakedomain", "api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.passivetotal.org/v2/account/quota", + json={"user": {"counts": {"search_api": 10}, "limits": {"search_api": 20}}}, + ) + module_test.httpx_mock.add_response( + url="https://api.passivetotal.org/v2/enrichment/subdomains?query=blacklanternsecurity.com", + json={"subdomains": ["asdf"]}, + ) + + def setup_after_prep(self, module_test): + module_test.monkeypatch.setattr(module_test.scan.modules["passivetotal"], "abort_if", lambda e: False) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_pgp.py b/bbot/test/test_step_1/module_tests/test_module_pgp.py index e69de29bb2..dac4152577 100644 --- a/bbot/test/test_step_1/module_tests/test_module_pgp.py +++ b/bbot/test/test_step_1/module_tests/test_module_pgp.py @@ -0,0 +1,35 @@ +from .base import ModuleTestBase + + +class TestPGP(ModuleTestBase): + web_body = """ + + +Search results for 'blacklanternsecurity.com' + + +

Search results for 'blacklanternsecurity.com'

Type bits/keyID            cr. time   exp time   key expir
+
+ + +
pub eddsa263/0xd4e98af823deadbeef 2022-09-14T15:11:31Z
+
+uid Asdf <asdf@blacklanternsecurity.com>
+sig  sig  0xd4e98af823deadbeef 2022-09-14T15:11:31Z 2024-09-14T17:00:00Z ____________________ [selfsig]
+
+
+""" + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=blacklanternsecurity.com", + text=self.web_body, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf@blacklanternsecurity.com" for e in events), "Failed to detect email" diff --git a/bbot/test/test_step_2/test_helpers.py b/bbot/test/test_step_2/test_helpers.py index 9639f6110c..ec91009b67 100644 --- a/bbot/test/test_step_2/test_helpers.py +++ b/bbot/test/test_step_2/test_helpers.py @@ -79,6 +79,12 @@ async def test_helpers(helpers, scan, bbot_scanner, bbot_config, bbot_httpserver ] assert helpers.is_ip("127.0.0.1") assert not helpers.is_ip("127.0.0.0.1") + assert helpers.is_dns_name("evilcorp.com") + assert helpers.is_dns_name("evilcorp") + assert helpers.is_dns_name("ドメイン.テスト") + assert not helpers.is_dns_name("127.0.0.1") + assert not helpers.is_dns_name("dead::beef") + assert not helpers.is_dns_name("bob@evilcorp.com") assert helpers.domain_stem("evilcorp.co.uk") == "evilcorp" assert helpers.domain_stem("www.evilcorp.co.uk") == "www.evilcorp" From ca55a854c9b50ca45ec7a2ead2ea2ad63ca9b825 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 21 May 2023 20:47:41 -0400 Subject: [PATCH 034/387] tests for riddler, rapiddns, python --- bbot/modules/rapiddns.py | 5 +++-- bbot/modules/riddler.py | 5 +++-- .../module_tests/test_module_python.py | 6 ++++++ .../module_tests/test_module_rapiddns.py | 16 ++++++++++++++++ .../module_tests/test_module_riddler.py | 16 ++++++++++++++++ 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py index 61bfa767dd..0af7e39306 100644 --- a/bbot/modules/rapiddns.py +++ b/bbot/modules/rapiddns.py @@ -9,9 +9,10 @@ class rapiddns(crobat): base_url = "https://rapiddns.io" - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result" - return self.request_with_fail_count(url) + response = await self.request_with_fail_count(url) + return response def parse_results(self, r, query): results = set() diff --git a/bbot/modules/riddler.py b/bbot/modules/riddler.py index a7128943c8..c6f865ee10 100644 --- a/bbot/modules/riddler.py +++ b/bbot/modules/riddler.py @@ -9,9 +9,10 @@ class riddler(crobat): base_url = "https://riddler.io" - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/search/exportcsv?q=pld:{self.helpers.quote(query)}" - return self.request_with_fail_count(url) + response = await self.request_with_fail_count(url) + return response def parse_results(self, r, query): results = set() diff --git a/bbot/test/test_step_1/module_tests/test_module_python.py b/bbot/test/test_step_1/module_tests/test_module_python.py index e69de29bb2..eb1628437b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_python.py +++ b/bbot/test/test_step_1/module_tests/test_module_python.py @@ -0,0 +1,6 @@ +from .base import ModuleTestBase + + +class TestPython(ModuleTestBase): + def check(self, module_test, events): + assert any(e.data == "blacklanternsecurity.com" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_rapiddns.py b/bbot/test/test_step_1/module_tests/test_module_rapiddns.py index e69de29bb2..e324a5ac14 100644 --- a/bbot/test/test_step_1/module_tests/test_module_rapiddns.py +++ b/bbot/test/test_step_1/module_tests/test_module_rapiddns.py @@ -0,0 +1,16 @@ +from .base import ModuleTestBase + + +class TestRapidDNS(ModuleTestBase): + web_body = """12 +asdf.blacklanternsecurity.com +asdf.blacklanternsecurity.com.""" + + def setup_after_prep(self, module_test): + module_test.module.abort_if = lambda e: False + module_test.httpx_mock.add_response( + url=f"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result", text=self.web_body + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_riddler.py b/bbot/test/test_step_1/module_tests/test_module_riddler.py index e69de29bb2..f06a0513b0 100644 --- a/bbot/test/test_step_1/module_tests/test_module_riddler.py +++ b/bbot/test/test_step_1/module_tests/test_module_riddler.py @@ -0,0 +1,16 @@ +from .base import ModuleTestBase + + +class TestRiddler(ModuleTestBase): + web_body = """12 +asdf.blacklanternsecurity.com +asdf.blacklanternsecurity.com.""" + + def setup_after_prep(self, module_test): + module_test.module.abort_if = lambda e: False + module_test.httpx_mock.add_response( + url=f"https://riddler.io/search/exportcsv?q=pld:blacklanternsecurity.com", text=self.web_body + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" From 29845eb21ac3669e5b1b898cd49f603f0229dd4b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 21 May 2023 21:33:38 -0400 Subject: [PATCH 035/387] skymem, shodan, securitytrails, secretsdb tests --- bbot/modules/securitytrails.py | 14 +- bbot/modules/shodan_dns.py | 8 +- bbot/modules/skymem.py | 11 +- .../module_tests/test_module_robots.py | 42 + .../module_tests/test_module_secretsdb.py | 22 + .../test_module_securitytrails.py | 21 + .../module_tests/test_module_shodan_dns.py | 21 + .../module_tests/test_module_skymem.py | 1222 +++++++++++++++++ 8 files changed, 1347 insertions(+), 14 deletions(-) diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 592616ae55..d58b227e50 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -11,18 +11,20 @@ class securitytrails(shodan_dns): base_url = "https://api.securitytrails.com/v1" - def setup(self): + async def setup(self): self.limit = 100 - return super().setup() + return await super().setup() - def ping(self): - r = self.request_with_fail_count(f"{self.base_url}/ping?apikey={self.api_key}") + async def ping(self): + url = f"{self.base_url}/ping?apikey={self.api_key}" + r = await self.request_with_fail_count(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}" - return self.request_with_fail_count(url) + response = await self.request_with_fail_count(url) + return response def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 5323e74a40..c94d0ac203 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -21,16 +21,18 @@ async def setup(self): return await self.require_api_key() async def ping(self): - r = await self.request_with_fail_count(f"{self.base_url}/api-info?key={self.api_key}") + url = f"{self.base_url}/api-info?key={self.api_key}" + r = await self.request_with_fail_count(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content async def request_url(self, query): url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}" - return await self.request_with_fail_count(url) + response = await self.request_with_fail_count(url) + return response def parse_results(self, r, query): json = r.json() if json: - for hostname in json.get("subdomains"): + for hostname in json.get("subdomains", []): yield f"{hostname}.{query}" diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py index e319cc6dba..56f5e39f65 100644 --- a/bbot/modules/skymem.py +++ b/bbot/modules/skymem.py @@ -11,14 +11,14 @@ class skymem(emailformat): base_url = "https://www.skymem.info" - def handle_event(self, event): + async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) # get first page url = f"{self.base_url}/srch?q={self.helpers.quote(query)}" - r = self.request_with_fail_count(url) + r = await self.request_with_fail_count(url) if not r: return - for email in self.extract_emails(r.text): + for email in self.helpers.extract_emails(r.text): self.emit_event(email, "EMAIL_ADDRESS", source=event) # iterate through other pages @@ -27,12 +27,13 @@ def handle_event(self, event): return domain_id = domain_ids[0] for page in range(2, 22): - r2 = self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}") + r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}") if not r2: continue - for email in self.extract_emails(r2.text): + for email in self.helpers.extract_emails(r2.text): self.emit_event(email, "EMAIL_ADDRESS", source=event) pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text) + self.critical(pages) if not pages: break last_page = max([int(p) for p in pages]) diff --git a/bbot/test/test_step_1/module_tests/test_module_robots.py b/bbot/test/test_step_1/module_tests/test_module_robots.py index e69de29bb2..aba68778fc 100644 --- a/bbot/test/test_step_1/module_tests/test_module_robots.py +++ b/bbot/test/test_step_1/module_tests/test_module_robots.py @@ -0,0 +1,42 @@ +import re +from .base import ModuleTestBase + + +class TestRobots(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "robots"] + config_overrides = {"modules": {"robots": {"include_sitemap": True}}} + + def setup_after_prep(self, module_test): + sample_robots = f"Allow: /allow/\nDisallow: /disallow/\nJunk: test.com\nDisallow: /*/wildcard.txt\nSitemap: {self.targets[0]}/sitemap.txt" + + expect_args = {"method": "GET", "uri": "/robots.txt"} + respond_args = {"response_data": sample_robots} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + allow_bool = False + disallow_bool = False + sitemap_bool = False + wildcard_bool = False + + for e in events: + if e.type == "URL_UNVERIFIED": + if str(e.module) != "TARGET": + assert "spider-danger" in e.tags, f"{e} doesn't have spider-danger tag" + if e.data == "http://127.0.0.1:8888/allow/": + allow_bool = True + + if e.data == "http://127.0.0.1:8888/disallow/": + disallow_bool = True + + if e.data == "http://127.0.0.1:8888/sitemap.txt": + sitemap_bool = True + + if re.match(r"http://127\.0\.0\.1:8888/\w+/wildcard\.txt", e.data): + wildcard_bool = True + + assert allow_bool + assert disallow_bool + assert sitemap_bool + assert wildcard_bool diff --git a/bbot/test/test_step_1/module_tests/test_module_secretsdb.py b/bbot/test/test_step_1/module_tests/test_module_secretsdb.py index e69de29bb2..5c0d1a00df 100644 --- a/bbot/test/test_step_1/module_tests/test_module_secretsdb.py +++ b/bbot/test/test_step_1/module_tests/test_module_secretsdb.py @@ -0,0 +1,22 @@ +from .base import ModuleTestBase + + +class TestSecretsDB(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "secretsdb"] + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml", + text="""patterns: +- pattern: + confidence: 99 + name: Asymmetric Private Key + regex: '-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'""", + ) + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "-----BEGIN PGP PRIVATE KEY BLOCK-----"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "FINDING" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_securitytrails.py b/bbot/test/test_step_1/module_tests/test_module_securitytrails.py index e69de29bb2..8e015baf88 100644 --- a/bbot/test/test_step_1/module_tests/test_module_securitytrails.py +++ b/bbot/test/test_step_1/module_tests/test_module_securitytrails.py @@ -0,0 +1,21 @@ +from .base import ModuleTestBase + + +class TestSecurityTrails(ModuleTestBase): + config_overrides = {"modules": {"securitytrails": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.securitytrails.com/v1/ping?apikey=asdf", + ) + module_test.httpx_mock.add_response( + url="https://api.securitytrails.com/v1/domain/blacklanternsecurity.com/subdomains?apikey=asdf", + json={ + "subdomains": [ + "asdf", + ], + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py b/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py index e69de29bb2..dfe19dfaec 100644 --- a/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py +++ b/bbot/test/test_step_1/module_tests/test_module_shodan_dns.py @@ -0,0 +1,21 @@ +from .base import ModuleTestBase + + +class TestShodan_DNS(ModuleTestBase): + config_overrides = {"modules": {"shodan_dns": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.shodan.io/api-info?key=asdf", + ) + module_test.httpx_mock.add_response( + url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf", + json={ + "subdomains": [ + "asdf", + ], + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_skymem.py b/bbot/test/test_step_1/module_tests/test_module_skymem.py index e69de29bb2..a7c7268b1a 100644 --- a/bbot/test/test_step_1/module_tests/test_module_skymem.py +++ b/bbot/test/test_step_1/module_tests/test_module_skymem.py @@ -0,0 +1,1222 @@ +from .base import ModuleTestBase + + +class TestSkymem(ModuleTestBase): + targets = ["blacklanternsecurity.com"] + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://www.skymem.info/srch?q=blacklanternsecurity.com", + text=page_1_body, + ) + module_test.httpx_mock.add_response( + url="https://www.skymem.info/domain/5679236812ad5b3f748a413d?p=2", + text=page_2_body, + ) + module_test.httpx_mock.add_response( + url="https://www.skymem.info/domain/5679236812ad5b3f748a413d?p=3", + text=page_3_body, + ) + + def check(self, module_test, events): + assert any(e.data == "page1email@blacklanternsecurity.com" for e in events), "Failed to detect first email" + assert any(e.data == "page2email@blacklanternsecurity.com" for e in events), "Failed to detect second email" + assert any(e.data == "page3email@blacklanternsecurity.com" for e in events), "Failed to detect third email" + + +page_1_body = """ + + + + + + + q=blacklanternsecurity.com - blacklanternsecurity.com=1768 emails + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+
\ + + + +
+
+
+ +
+
+ + +
+ +
+
+ +
+
+
+
+
+ + + + + +
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+ + +
+ + +
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + +""" +page_2_body = """ + + + + + + q=blacklanternsecurity.com - blacklanternsecurity.com=1768 emails + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+
\ + + + +
+
+
+ +
+
+ + +
+ +
+
+ +
+
+
+
+
+ + + + + +
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+ + +
+ + +
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + +""" +page_3_body = """ + + + + + + q=blacklanternsecurity.com - blacklanternsecurity.com=1768 emails + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+
\ + + + +
+
+
+ +
+
+ + +
+ +
+
+ +
+
+
+
+
+ + + + + +
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+ + +
+ + +
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + +""" From 516bf8d3ad97a0ad4321d37932f002d2ea328150 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 May 2023 01:18:00 -0400 Subject: [PATCH 036/387] tests for ntlm, social, speculate, sublist3r, telerik, threatminer, url_manipulation, urlscan, viewdns, virustotal, wafw00f, wappalyzer --- bbot/core/helpers/web.py | 5 +- bbot/modules/ntlm.py | 3 +- bbot/modules/threatminer.py | 8 +- bbot/modules/url_manipulation.py | 2 +- bbot/modules/urlscan.py | 21 +- bbot/modules/viewdns.py | 13 +- bbot/modules/virustotal.py | 11 +- bbot/modules/wappalyzer.py | 8 +- .../module_tests/test_module_social.py | 14 + .../module_tests/test_module_speculate.py | 22 ++ .../test_module_subdomain_hijack.py | 44 +++ .../module_tests/test_module_sublist3r.py | 13 + .../module_tests/test_module_telerik.py | 74 ++++ .../module_tests/test_module_threatminer.py | 12 + .../test_module_url_manipulation.py | 39 ++ .../module_tests/test_module_urlscan.py | 58 +++ .../module_tests/test_module_viewdns.py | 333 ++++++++++++++++++ .../module_tests/test_module_virustotal.py | 51 +++ .../module_tests/test_module_wafw00f.py | 14 + .../module_tests/test_module_wappalyzer.py | 20 ++ 20 files changed, 733 insertions(+), 32 deletions(-) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index d47b6289f2..f111071273 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -119,7 +119,7 @@ async def download(self, url, **kwargs): cache_hrs = float(kwargs.pop("cache_hrs", -1)) log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}") if cache_hrs > 0 and self.parent_helper.is_cached(url): - log.debug(f"{url} is cached") + log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}") success = True else: # kwargs["raise_error"] = True @@ -175,9 +175,6 @@ async def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs): offset = 0 while 1: new_url = url.format(page=page, page_size=page_size, offset=offset) - data = requests_kwargs.get("data", None) - if data is not None: - requests_kwargs["data"] = requests_kwargs["data"].format(page=page, page_size=page_size, offset=offset) result = await self.request(new_url, **requests_kwargs) try: if json: diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index a5d2fed372..6f5f535d0e 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -3,7 +3,8 @@ ntlm_discovery_endpoints = [ "", - "autodiscover/autodiscover.xml" "ecp/", + "autodiscover/autodiscover.xml", + "ecp/", "ews/", "ews/exchange.asmx", "exchange/", diff --git a/bbot/modules/threatminer.py b/bbot/modules/threatminer.py index 3f6c99f2fe..066e4c3bb1 100644 --- a/bbot/modules/threatminer.py +++ b/bbot/modules/threatminer.py @@ -11,9 +11,11 @@ class threatminer(crobat): base_url = "https://api.threatminer.org/v2" - def request_url(self, query): - return self.request_with_fail_count(f"{self.base_url}/domain.php?q={self.helpers.quote(query)}&rt=5") + async def request_url(self, query): + url = f"{self.base_url}/domain.php?q={self.helpers.quote(query)}&rt=5" + r = await self.request_with_fail_count(url) + return r def parse_results(self, r, query): j = r.json() - yield from j.get("results", []) + return list(j.get("results", [])) diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index 851761dc2a..1af1316ccc 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -78,7 +78,7 @@ async def handle_event(self, event): else: self.debug("Ignoring positive result due to presence of parameter name in result") - def filter_event(self, event): + async def filter_event(self, event): accepted_status_codes = ["200", "301", "302"] for c in accepted_status_codes: diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index 2b3b92400d..320c327618 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -13,13 +13,13 @@ class urlscan(crobat): base_url = "https://urlscan.io/api/v1" - def setup(self): + async def setup(self): self.urls = self.config.get("urls", False) - return super().setup() + return await super().setup() - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - for domain, url in self.query(query): + for domain, url in await self.query(query): source_event = event if domain and domain != query: domain_event = self.make_event(domain, "DNS_NAME", source=event) @@ -38,10 +38,12 @@ def handle_event(self, event): else: self.debug(f"{url_event.host} does not match {query}") - def query(self, query): - results = self.helpers.request(f"{self.base_url}/search/?q={self.helpers.quote(query)}") + async def query(self, query): + results = set() + url = f"{self.base_url}/search/?q={self.helpers.quote(query)}" + r = await self.helpers.request(url) try: - json = results.json() + json = r.json() if json and type(json) == dict: for result in json.get("results", []): if result and type(result) == dict: @@ -50,14 +52,15 @@ def query(self, query): domain = task.get("domain", "") url = task.get("url", "") if domain or url: - yield domain, url + results.add((domain, url)) page = result.get("page", {}) if page and type(page) == dict: domain = page.get("domain", "") url = page.get("url", "") if domain or url: - yield domain, url + results.add((domain, url)) else: self.debug(f'No results for "{query}"') except Exception: self.verbose(f"Error retrieving urlscan results") + return results diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py index 0714bdc8e5..8ddc3e818c 100644 --- a/bbot/modules/viewdns.py +++ b/bbot/modules/viewdns.py @@ -39,6 +39,7 @@ async def handle_event(self, event): # todo: registrar? async def query(self, query): + results = set() url = f"{self.base_url}/reversewhois/?q={query}" r = await self.helpers.request(url) status_code = getattr(r, "status_code", 0) @@ -47,7 +48,7 @@ async def query(self, query): content = getattr(r, "content", b"") html = BeautifulSoup(content, features="lxml") - yielded = set() + found = set() for table_row in html.findAll("tr"): table_cells = table_row.findAll("td") # make double-sure we're in the right table by checking the date field @@ -58,10 +59,12 @@ async def query(self, query): # registrar == last cell registrar = table_cells[-1].text.strip() if domain and not domain == query: - to_yield = (domain, registrar) - to_yield_hash = hash(to_yield) - if to_yield_hash not in yielded: - yield to_yield + result = (domain, registrar) + result_hash = hash(result) + if result_hash not in found: + found.add(result_hash) + results.add(result) except IndexError: self.debug(f"Invalid row {str(table_row)[:40]}...") continue + return results diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 1a15985620..dac8ee6335 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -11,18 +11,19 @@ class virustotal(shodan_dns): base_url = "https://www.virustotal.com/api/v3" - def setup(self): + async def setup(self): self.api_key = self.config.get("api_key", "") self.headers = {"x-apikey": self.api_key} - return super().setup() + return await super().setup() - def ping(self): + async def ping(self): # virustotal does not have a ping function return - def request_url(self, query): + async def request_url(self, query): url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains" - return self.request_with_fail_count(url, headers=self.headers) + r = await self.request_with_fail_count(url, headers=self.headers) + return r def parse_results(self, r, query): results = set() diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index 5d4e51fa54..a372d1791d 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -22,12 +22,12 @@ class wappalyzer(BaseModule): scope_distance_modifier = None max_event_handlers = 5 - def setup(self): - self.wappalyzer = Wappalyzer.latest() + async def setup(self): + self.wappalyzer = await self.scan.run_in_executor(Wappalyzer.latest) return True - def handle_event(self, event): - for res in self.wappalyze(event.data): + async def handle_event(self, event): + for res in await self.scan.run_in_executor(self.wappalyze, event.data): self.emit_event( {"technology": res.lower(), "url": event.data["url"], "host": str(event.host)}, "TECHNOLOGY", event ) diff --git a/bbot/test/test_step_1/module_tests/test_module_social.py b/bbot/test/test_step_1/module_tests/test_module_social.py index e69de29bb2..caded2ec78 100644 --- a/bbot/test/test_step_1/module_tests/test_module_social.py +++ b/bbot/test/test_step_1/module_tests/test_module_social.py @@ -0,0 +1,14 @@ +from .base import ModuleTestBase + + +class TestSocial(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "social"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": ''} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "SOCIAL" and e.data["platform"] == "discord" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_speculate.py b/bbot/test/test_step_1/module_tests/test_module_speculate.py index e69de29bb2..27156707bf 100644 --- a/bbot/test/test_step_1/module_tests/test_module_speculate.py +++ b/bbot/test/test_step_1/module_tests/test_module_speculate.py @@ -0,0 +1,22 @@ +from .base import ModuleTestBase + + +class TestSpeculate_Subdirectories(ModuleTestBase): + targets = ["http://127.0.0.1:8888/subdir1/subdir2/"] + modules_overrides = ["httpx", "speculate"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/subdir1/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/subdir1/subdir2/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/subdir1/" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py b/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py index e69de29bb2..c0c1006279 100644 --- a/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py +++ b/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py @@ -0,0 +1,44 @@ +from .base import ModuleTestBase + + +class TestSubdomain_Hijack(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "subdomain_hijack"] + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://raw.githubusercontent.com/EdOverflow/can-i-take-over-xyz/master/fingerprints.json", + json=[ + { + "cicd_pass": True, + "cname": ["us-east-1.elasticbeanstalk.com"], + "discussion": "[Issue #194](https://github.com/EdOverflow/can-i-take-over-xyz/issues/194)", + "documentation": "", + "fingerprint": "NXDOMAIN", + "http_status": None, + "nxdomain": True, + "service": "AWS/Elastic Beanstalk", + "status": "Vulnerable", + "vulnerable": True, + } + ], + ) + + def setup_after_prep(self, module_test): + fingerprints = module_test.module.fingerprints + assert fingerprints, "No subdomain hijacking fingerprints available" + fingerprint = next(iter(fingerprints)) + rand_string = module_test.scan.helpers.rand_string(length=15, digits=False) + self.rand_subdomain = f"{rand_string}.{next(iter(fingerprint.domains))}" + module_test.log.critical(self.rand_subdomain) + respond_args = {"response_data": f''} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any( + event.type == "FINDING" + and event.data["description"].startswith("Hijackable Subdomain") + and self.rand_subdomain in event.data["description"] + and event.data["host"] == self.rand_subdomain + for event in events + ), f"No hijackable subdomains in {events}" diff --git a/bbot/test/test_step_1/module_tests/test_module_sublist3r.py b/bbot/test/test_step_1/module_tests/test_module_sublist3r.py index e69de29bb2..a7cacd2310 100644 --- a/bbot/test/test_step_1/module_tests/test_module_sublist3r.py +++ b/bbot/test/test_step_1/module_tests/test_module_sublist3r.py @@ -0,0 +1,13 @@ +from .base import ModuleTestBase + + +class TestSublist3r(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://api.sublist3r.com/search.php?domain=blacklanternsecurity.com", + json=["asdf.blacklanternsecurity.com", "zzzz.blacklanternsecurity.com"], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "zzzz.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_telerik.py b/bbot/test/test_step_1/module_tests/test_module_telerik.py index e69de29bb2..3e6dba2299 100644 --- a/bbot/test/test_step_1/module_tests/test_module_telerik.py +++ b/bbot/test/test_step_1/module_tests/test_module_telerik.py @@ -0,0 +1,74 @@ +import re +from .base import ModuleTestBase + + +class TestTelerik(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "telerik"] + config_overrides = {"modules": {"telerik": {"exploit_RAU_crypto": True}}} + + def setup_before_prep(self, module_test): + # Simulate Telerik.Web.UI.WebResource.axd?type=rau detection + expect_args = {"method": "GET", "uri": "/Telerik.Web.UI.WebResource.axd", "query_string": "type=rau"} + respond_args = { + "response_data": '{ "message" : "RadAsyncUpload handler is registered succesfully, however, it may not be accessed directly." }' + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # Simulate Vulnerable Telerik.Web.UI.WebResource.axd + vuln_data = "ATTu5i4R+ViNFYO6kst0jC11wM/1iqH+W/isjhaDjNuCI7eJ/BY5d1E9eqZK27CJCMuon9u8/hgRIM/cTlgLlv4qOYjPBjs81Y3dAZAdtIr3TXiCmZi9M09a1BYMxjvGKfVky3b7PoOppeWS/3rglTwL1e8oyqLGx2NKUH5y8Cd+kLKV2f31J1sV4I5HTDKgDmvziJp3zlDrCb0Fi9ilKH+O1cbVx6SdBop/U30FxLaB/QIbt2N1rQHREJ5Skpgo7dilPxzBaTObdBhCVyB/FiJhenS/0u3h0Mpi6+A40SylICcyyxQha7+Uh7lEJ8Ne+2eTs4WqcaaQbvIhy7oHc+D0soxRKMZRjo7Up+UWHQJJh6KtWSCxUESNSdNcxjPQZE9HqsPlldVlkeC+ehSGce5bR0Ylots6Iz1OoCgMEWwxByeG3VzgxF6XpitL61A1hFcNo9euSTnCfOWh0vrQHON7DN5LpM9xr7SoD0Dnu01hZ9NS1PHhPLyN5WS87u5qdZp/z3Sxwc3wawIdo62RNf4Iz2gAKJZnPfxrE1mRn5kBe7f6O44rcuv6lcdao/DGlwbERKwRI6/n+FxGmc7H5iEKyihIwS2XUoOgsYTx5CWCDM8CuOXTk+H5fPYp9APRPbkD1IS9I/vRmvNPwWsgv8/7DzttqdBsGxiZJfCw1uZ7KSVmbItgXPAcscNxGEMaHXyJzkAl/mlM5/t/YSejwYoSW6jFfQcLdaVx2dpIpl5UmmQjFedzKeiNqpZDCk4yzXFHX24XUODYMJDtIJK2Hz1KTZmFG+LAOJjB9QOI58hFAnytcKay+JWFrzah/IvoNZxJUtlYdxw0YEyKs/ExET7AXgYQN0S+8j2PfaMMpzDSctTqpp5XBFV4Mt718GiqVnQJtWQv2p9Xl8XXOerBthbzzAciVcB8AV2WfZ51W3e4aX4kcyT/sCJhm7NR5WrNG5mX/ns0TTnGnzlPYhJcbu8uMFjMGDpXuhVyroJ7wmZucaIvesg0h5Y9cMEFviqsdy15vjMzFh+v9uO9Vicf6n9Z9JGSpWKE8wer2JU5b53Zw0cTfulAAffLWXnzOnfu&6R/cGaqQeHVAzdJ9wTFOyCsrMSTtqcjLe8AHwiPckPDUwecnJyNlkDYwDQpxGYQ9hs6YxhupK310sbCbtXB4H6Dz5rGNL40nkkyo4j2clmRr08jtFsPQ0RpE5BGsulPT3l0MxyAvPFMs8bMybUyAP+9RB9LoHE3Xo8BqDadX3HQakpPfGtiDMp+wxkWRgaNpCnXeY1QewWTF6z/duLzbu6CT6s+H4HgBHrOLTpemC2PvP2bDm0ySPHLdpapLYxU8nIYjLKIyYJgwv9S9jNckIVpcGVTWVul7CauCKxAB2mMnM9jJi8zfFwKajT5d2d9XfpkiVMrdlmikSB/ehyX1wQ==" + expect_args = { + "method": "POST", + "uri": "/Telerik.Web.UI.WebResource.axd", + "query_string": "type=rau", + "data": vuln_data, + } + respond_args = { + "response_data": '{"fileInfo":{"FileName":"RAU_crypto.bypass","ContentType":"text/html","ContentLength":5,"DateJson":"2019-01-02T03:04:05.067Z","Index":0}, "metaData":"CS8S/Z0J/b2982DRxDin0BBslA7fI0cWMuWlPu4W3FkE4tKaVoIEiAOtVlJ6D+0RQsfu8ox6gvMYxceQ0LtWyTkQBaIUa8LgLQg05DMaQuufHNx0YQ2ACi5neqDBvduj2MGiSGC0hNKzSWsHystZGUfFPLTZuJXYnff+WXurecuRzSI7d4Q1aj0bcTKKvfyQtH+fsTEafWRRZ99X/xgi4ON2OsRZ738uQHw7pQT2e1v7AtN46mxO/BmhEuZQr6m6HEvxK0pJRNkBhFUiQ+poeu8j3JzicOjvPDwFE4Rjqf3RVILt83XZrju2VpRIJqAEtf//znhH8BhT5BWvhnRo+J3ML5qoZLa2joE/QK8Ctf3UPvAFkHIUMdOH2mLNgZ+U87tdVE6fYfzvphZsLxmJRG45H8ZTZuYhJbOfei2LQ4fqHmr7p8KpJNVqoz/ev1dnBclAf5ayb40qJKEVsGXIbWEbIZwg7TTsLFc29aP7DPg=" }' + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # Simulate DialogHandler detection + expect_args = {"method": "GET", "uri": "Telerik.Web.UI.SpellCheckHandler.axd"} + respond_args = {"status": 500} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # Simulate DialogHandler detection + expect_args = {"method": "GET", "uri": "/App_Master/Telerik.Web.UI.DialogHandler.aspx"} + respond_args = { + "response_data": '
Cannot deserialize dialog parameters. Please refresh the editor page.
Error Message:Invalid length for a Base-64 char array or string.
' + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + # Fallback + expect_args = {"uri": re.compile(r"^/\w{10}$")} + respond_args = {"status": 200} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + telerik_axd_detection = False + telerik_axd_vulnerable = False + telerik_spellcheck_detection = False + telerik_dialoghandler_detection = False + + for e in events: + if e.type == "FINDING" and "Telerik RAU AXD Handler detected" in e.data["description"]: + telerik_axd_detection = True + continue + + if e.type == "VULNERABILITY" and "Confirmed Vulnerable Telerik (version: 2014.3.1024)": + telerik_axd_vulnerable = True + continue + + if e.type == "FINDING" and "Telerik DialogHandler detected" in e.data["description"]: + telerik_dialoghandler_detection = True + continue + + if e.type == "FINDING" and "Telerik SpellCheckHandler detected" in e.data["description"]: + telerik_spellcheck_detection = True + continue + + assert telerik_axd_detection, "Telerik AXD detection failed" + assert telerik_axd_vulnerable, "Telerik vulnerable AXD detection failed" + assert telerik_spellcheck_detection, "Telerik spellcheck detection failed" + assert telerik_dialoghandler_detection, "Telerik dialoghandler detection failed" diff --git a/bbot/test/test_step_1/module_tests/test_module_threatminer.py b/bbot/test/test_step_1/module_tests/test_module_threatminer.py index e69de29bb2..71864f1b25 100644 --- a/bbot/test/test_step_1/module_tests/test_module_threatminer.py +++ b/bbot/test/test_step_1/module_tests/test_module_threatminer.py @@ -0,0 +1,12 @@ +from .base import ModuleTestBase + + +class TestThreatminer(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.threatminer.org/v2/domain.php?q=blacklanternsecurity.com&rt=5", + json={"results": ["asdf.blacklanternsecurity.com"]}, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py b/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py index e69de29bb2..8a197dbe7b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py +++ b/bbot/test/test_step_1/module_tests/test_module_url_manipulation.py @@ -0,0 +1,39 @@ +from .base import ModuleTestBase + + +class TestUrl_Manipulation(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "url_manipulation"] + body = """ + + the title + +

Hello null!

'; + + + """ + + body_match = """ + + the title + +

Hello AAAAAAAAAAAAAA!

'; + + + """ + + def setup_after_prep(self, module_test): + expect_args = {"query_string": f"{module_test.module.rand_string}=.xml".encode()} + respond_args = {"response_data": self.body_match} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + respond_args = {"response_data": self.body} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any( + e.type == "FINDING" + and e.data["description"] + == f"Url Manipulation: [body] Sig: [Modified URL: http://127.0.0.1:8888/?{module_test.module.rand_string}=.xml]" + for e in events + ) diff --git a/bbot/test/test_step_1/module_tests/test_module_urlscan.py b/bbot/test/test_step_1/module_tests/test_module_urlscan.py index e69de29bb2..51ec290fcf 100644 --- a/bbot/test/test_step_1/module_tests/test_module_urlscan.py +++ b/bbot/test/test_step_1/module_tests/test_module_urlscan.py @@ -0,0 +1,58 @@ +from .base import ModuleTestBase + + +class TestUrlScan(ModuleTestBase): + config_overrides = {"modules": {"urlscan": {"urls": True}}} + + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://urlscan.io/api/v1/search/?q=blacklanternsecurity.com", + json={ + "results": [ + { + "task": { + "visibility": "public", + "method": "api", + "domain": "asdf.blacklanternsecurity.com", + "apexDomain": "blacklanternsecurity.com", + "time": "2023-05-17T01:45:11.391Z", + "uuid": "c558b3b3-b274-4339-99ef-301eb043741f", + "url": "https://asdf.blacklanternsecurity.com/cna.html", + }, + "stats": { + "uniqIPs": 6, + "uniqCountries": 3, + "dataLength": 926713, + "encodedDataLength": 332213, + "requests": 22, + }, + "page": { + "country": "US", + "server": "GitHub.com", + "ip": "2606:50c0:8002::153", + "mimeType": "text/html", + "title": "Vulnerability Program | Black Lantern Security", + "url": "https://asdf.blacklanternsecurity.com/cna.html", + "tlsValidDays": 89, + "tlsAgeDays": 25, + "tlsValidFrom": "2023-04-21T19:16:58.000Z", + "domain": "asdf.blacklanternsecurity.com", + "apexDomain": "blacklanternsecurity.com", + "asnname": "FASTLY, US", + "asn": "AS54113", + "tlsIssuer": "R3", + "status": "200", + }, + "_id": "c558b3b3-b274-4339-99ef-301eb043741f", + "_score": None, + "sort": [1684287911391, "c558b3b3-b274-4339-99ef-301eb043741f"], + "result": "https://urlscan.io/api/v1/result/c558b3b3-b274-4339-99ef-301eb043741f/", + "screenshot": "https://urlscan.io/screenshots/c558b3b3-b274-4339-99ef-301eb043741f.png", + } + ] + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "https://asdf.blacklanternsecurity.com/cna.html" for e in events), "Failed to detect URL" diff --git a/bbot/test/test_step_1/module_tests/test_module_viewdns.py b/bbot/test/test_step_1/module_tests/test_module_viewdns.py index e69de29bb2..d40960e483 100644 --- a/bbot/test/test_step_1/module_tests/test_module_viewdns.py +++ b/bbot/test/test_step_1/module_tests/test_module_viewdns.py @@ -0,0 +1,333 @@ +from .base import ModuleTestBase + + +class TestViewDNS(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://viewdns.info/reversewhois/?q=blacklanternsecurity.com", + text=web_body, + ) + + def check(self, module_test, events): + assert any( + e.data == "hyperloop.com" and "affiliate" in e.tags for e in events + ), "Failed to detect affiliate domain" + + +web_body = """ + + + Reverse Whois Lookup - ViewDNS.info + + + + + + + + + + + + + + + + + + + + + + + + + + +
ViewDNS.info - Your one source for DNS related tools! + + + + +
+ + + + + + + + + + + + + + + +
+ + ViewDNS.info > Tools > +

Reverse Whois Lookup

+

This free tool will allow you to find domain names owned by an individual person or company. Simply enter the email address or name of the person or company to find other domains registered using those same details. FAQ.

+ +
Registrant Name or Email Address:
+ +
+
+ Reverse Whois results for blacklanternsecurity.com
==============

There are 20 domains that matched this search query.
These are listed below:

+ + + + + + +
hyperloop.com2003-12-04NETWORK SOLUTIONS, LLC.
+
+
+ + + + + +
+ + + + +

+ +
+ + + + +
+
+ All content © 2023 ViewDNS.info
Feedback / Suggestions / Contact Us - Privacy Policy +
+
+ + + + +
+ + + +
+
+
+
+
+ + +""" diff --git a/bbot/test/test_step_1/module_tests/test_module_virustotal.py b/bbot/test/test_step_1/module_tests/test_module_virustotal.py index e69de29bb2..88663bdfaa 100644 --- a/bbot/test/test_step_1/module_tests/test_module_virustotal.py +++ b/bbot/test/test_step_1/module_tests/test_module_virustotal.py @@ -0,0 +1,51 @@ +from .base import ModuleTestBase + + +class TestVirusTotal(ModuleTestBase): + config_overrides = {"modules": {"virustotal": {"api_key": "asdf"}}} + + def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains", + json={ + "meta": {"count": 25, "cursor": "eyJsaW1pdCI6IDEwLCAib2Zmc2V0IjogMTB9"}, + "data": [ + { + "attributes": { + "last_dns_records": [{"type": "A", "value": "168.62.180.225", "ttl": 3600}], + "whois": "Creation Date: 2013-07-30T20:14:50Z\nDNSSEC: unsigned\nDomain Name: BLACKLANTERNSECURITY.COM\nDomain Status: clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited\nDomain Status: clientRenewProhibited https://icann.org/epp#clientRenewProhibited\nDomain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited\nDomain Status: clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited\nName Server: NS01.DOMAINCONTROL.COM\nName Server: NS02.DOMAINCONTROL.COM\nRegistrar Abuse Contact Email: abuse@godaddy.com\nRegistrar Abuse Contact Phone: 480-624-2505\nRegistrar IANA ID: 146\nRegistrar URL: http://www.godaddy.com\nRegistrar WHOIS Server: whois.godaddy.com\nRegistrar: GoDaddy.com, LLC\nRegistry Domain ID: 1818679075_DOMAIN_COM-VRSN\nRegistry Expiry Date: 2023-07-30T20:14:50Z\nUpdated Date: 2022-09-14T16:28:14Z", + "tags": [], + "popularity_ranks": {}, + "last_dns_records_date": 1657734301, + "last_analysis_stats": { + "harmless": 0, + "malicious": 0, + "suspicious": 0, + "undetected": 86, + "timeout": 0, + }, + "creation_date": 1375215290, + "reputation": 0, + "registrar": "GoDaddy.com, LLC", + "last_analysis_results": {}, + "last_update_date": 1663172894, + "last_modification_date": 1657734301, + "tld": "com", + "categories": {}, + "total_votes": {"harmless": 0, "malicious": 0}, + }, + "type": "domain", + "id": "asdf.blacklanternsecurity.com", + "links": {"self": "https://www.virustotal.com/api/v3/domains/asdf.blacklanternsecurity.com"}, + "context_attributes": {"timestamp": 1657734301}, + } + ], + "links": { + "self": "https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains?limit=10", + "next": "https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains?cursor=eyJsaW1pdCI6IDEwLCAib2Zmc2V0IjogMTB9&limit=10", + }, + }, + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_wafw00f.py b/bbot/test/test_step_1/module_tests/test_module_wafw00f.py index e69de29bb2..2b69c8b894 100644 --- a/bbot/test/test_step_1/module_tests/test_module_wafw00f.py +++ b/bbot/test/test_step_1/module_tests/test_module_wafw00f.py @@ -0,0 +1,14 @@ +from .base import ModuleTestBase + + +class TestWafw00f(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "wafw00f"] + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "Proudly powered by litespeed web server"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py b/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py index e69de29bb2..47e456b860 100644 --- a/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py +++ b/bbot/test/test_step_1/module_tests/test_module_wappalyzer.py @@ -0,0 +1,20 @@ +from .base import ModuleTestBase + + +class TestWappalyzer(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "wappalyzer"] + + def setup_after_prep(self, module_test): + respond_args = { + "response_data": """BBOT is life + + + +""", + "headers": {"Server": "Apache/2.4.41 (Ubuntu)"}, + } + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.type == "TECHNOLOGY" and e.data["technology"].lower() == "google font api" for e in events) From 6fdb598e2e6bcfbb7d3c47106f7c98a4d6aa2908 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 May 2023 01:20:26 -0400 Subject: [PATCH 037/387] removed services test --- bbot/test/test_step_1/module_tests/test_module_services.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 bbot/test/test_step_1/module_tests/test_module_services.py diff --git a/bbot/test/test_step_1/module_tests/test_module_services.py b/bbot/test/test_step_1/module_tests/test_module_services.py deleted file mode 100644 index e69de29bb2..0000000000 From b8000f4ffccced1fca4518e3de16fb5f1ec0e777 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 May 2023 12:53:46 -0400 Subject: [PATCH 038/387] more module tests! --- bbot/modules/bucket_gcp.py | 1 - bbot/modules/builtwith.py | 1 - bbot/modules/emailformat.py | 1 - bbot/modules/leakix.py | 1 - bbot/modules/output/web_report.py | 7 +- bbot/modules/skymem.py | 1 - bbot/modules/smuggler.py | 6 +- bbot/modules/wayback.py | 18 ++--- bbot/modules/zoomeye.py | 35 ++++++---- bbot/test/conftest.py | 4 +- bbot/test/test_step_1/module_tests/base.py | 10 --- .../module_tests/test_module_aggregate.py | 1 - .../module_tests/test_module_dnscommonsrv.py | 13 ++-- .../test_module_dnszonetransfer.py | 2 +- .../module_tests/test_module_generic_ssrf.py | 5 +- .../module_tests/test_module_host_header.py | 5 +- .../module_tests/test_module_ipneighbor.py | 23 ++++--- .../module_tests/test_module_smuggler.py | 9 +++ .../test_module_subdomain_hijack.py | 1 - .../module_tests/test_module_vhost.py | 65 +++++++++++++++++++ .../module_tests/test_module_wayback.py | 12 ++++ .../module_tests/test_module_web_report.py | 65 +++++++++++++++++++ .../module_tests/test_module_websocket.py | 35 ++++++++++ .../module_tests/test_module_zoomeye.py | 36 ++++++++++ 24 files changed, 296 insertions(+), 61 deletions(-) diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py index d4ac880a9c..b7e96d5b1d 100644 --- a/bbot/modules/bucket_gcp.py +++ b/bbot/modules/bucket_gcp.py @@ -50,5 +50,4 @@ async def check_bucket_exists(self, bucket_name, url): response = await self.helpers.request(url) status_code = getattr(response, "status_code", 0) existent_bucket = status_code not in (0, 400, 404) - self.critical(f"{bucket_name}: {url}: {existent_bucket}") return existent_bucket, set(), bucket_name, url diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 6fabcd2833..6c5a305c1a 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -61,7 +61,6 @@ def parse_domains(self, r, query): query (string): The query used against the API """ results_set = set() - self.critical(r.content) json = r.json() if json: results = json.get("Results", []) diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py index f28e19f9d9..82b5797445 100644 --- a/bbot/modules/emailformat.py +++ b/bbot/modules/emailformat.py @@ -14,7 +14,6 @@ async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) url = f"{self.base_url}/d/{self.helpers.quote(query)}/" r = await self.request_with_fail_count(url) - self.hugesuccess(r.content) if not r: return for email in self.helpers.extract_emails(r.text): diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 6622242e83..4ebf895703 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -31,7 +31,6 @@ async def ping(self): async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" response = await self.request_with_fail_count(url, headers=self.headers) - self.hugewarning(response.json()) return response def parse_results(self, r, query=None): diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index da973a6e38..793f26c329 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -32,7 +32,7 @@ async def setup(self): self._prep_output_dir("web_report.html") return True - def handle_event(self, event): + async def handle_event(self, event): if event.type == "URL": parsed = event.parsed host = f"{parsed.scheme}://{parsed.netloc}/" @@ -74,7 +74,7 @@ def handle_event(self, event): else: self.web_assets[host][event.type].append(html.escape(event.pretty_string)) - def report(self): + async def report(self): for host in self.web_assets.keys(): self.markdown += f"# {host}\n\n" @@ -93,5 +93,4 @@ def report(self): self.file.write(markdown.markdown(self.markdown)) self.file.write(self.html_footer) self.file.flush() - with self._report_lock: - self.info(f"Web Report saved to {self.output_file}") + self.info(f"Web Report saved to {self.output_file}") diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py index 56f5e39f65..71d0e883e7 100644 --- a/bbot/modules/skymem.py +++ b/bbot/modules/skymem.py @@ -33,7 +33,6 @@ async def handle_event(self, event): for email in self.helpers.extract_emails(r2.text): self.emit_event(email, "EMAIL_ADDRESS", source=event) pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text) - self.critical(pages) if not pages: break last_page = max([int(p) for p in pages]) diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py index 7284ccec60..f478a63641 100644 --- a/bbot/modules/smuggler.py +++ b/bbot/modules/smuggler.py @@ -23,11 +23,11 @@ class smuggler(BaseModule): } ] - def setup(self): + async def setup(self): self.scanned_hosts = set() return True - def handle_event(self, event): + async def handle_event(self, event): host = f"{event.parsed.scheme}://{event.parsed.netloc}/" host_hash = hash(host) if host_hash in self.scanned_hosts: @@ -44,7 +44,7 @@ def handle_event(self, event): "-u", event.data, ] - for f in self.helpers.run_live(command): + async for f in self.helpers.run_live(command): if "Issue Found" in f: technique = f.split(":")[0].rstrip() text = f.split(":")[1].split("-")[0].strip() diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 6ce7751526..133dc27e7b 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -17,19 +17,20 @@ class wayback(crobat): base_url = "http://web.archive.org" - def setup(self): + async def setup(self): self.urls = self.config.get("urls", False) self.garbage_threshold = self.config.get("garbage_threshold", 10) - return super().setup() + return await super().setup() - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - for result, event_type in self.query(query): + for result, event_type in await self.query(query): self.emit_event(result, event_type, event, abort_if=self.abort_if) - def query(self, query): + async def query(self, query): + results = set() waybackurl = f"{self.base_url}/cdx/search/cdx?url={self.helpers.quote(query)}&matchType=domain&output=json&fl=original&collapse=original" - r = self.helpers.request(waybackurl, timeout=self.http_timeout + 10) + r = await self.helpers.request(waybackurl, timeout=self.http_timeout + 10) if not r: self.warning(f'Error connecting to archive.org for query "{query}"') return @@ -55,6 +56,7 @@ def query(self, query): h = hash(dns_name) if h not in dns_names: dns_names.add(h) - yield dns_name, "DNS_NAME" + results.add((dns_name, "DNS_NAME")) else: - yield parsed_url.geturl(), "URL_UNVERIFIED" + results.add((parsed_url.geturl(), "URL_UNVERIFIED")) + return results diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py index 5ebfad9fef..83f9bd641f 100644 --- a/bbot/modules/zoomeye.py +++ b/bbot/modules/zoomeye.py @@ -15,19 +15,20 @@ class zoomeye(shodan_dns): base_url = "https://api.zoomeye.org" - def setup(self): + async def setup(self): self.max_pages = self.config.get("max_pages", 20) self.headers = {"API-KEY": self.config.get("api_key", "")} self.include_related = self.config.get("include_related", False) - return super().setup() + return await super().setup() - def ping(self): - r = self.helpers.request(f"{self.base_url}/resources-info", headers=self.headers) + async def ping(self): + url = f"{self.base_url}/resources-info" + r = await self.helpers.request(url, headers=self.headers) assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining" - def handle_event(self, event): + async def handle_event(self, event): query = self.make_query(event) - results = self.query(query) + results = await self.query(query) if results: for hostname in results: if hostname == event: @@ -37,15 +38,23 @@ def handle_event(self, event): tags = ["affiliate"] self.emit_event(hostname, "DNS_NAME", event, tags=tags) - def query(self, query): + async def query(self, query): + results = set() query_type = 0 if self.include_related else 1 url = f"{self.base_url}/domain/search?q={self.helpers.quote(query)}&type={query_type}&page=" + "{page}" - for i, j in enumerate(self.helpers.api_page_iter(url, headers=self.headers)): - results = list(self.parse_results(j)) - if results: - yield from results - if not results or i >= (self.max_pages - 1) or self.scan.stopping: - break + i = 0 + agen = self.helpers.api_page_iter(url, headers=self.headers) + try: + async for j in agen: + r = list(self.parse_results(j)) + if r: + results.update(set(r)) + if not r or i >= (self.max_pages - 1): + break + i += 1 + finally: + agen.aclose() + return results def parse_results(self, r): for entry in r.get("list", []): diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 0aca0ff1bd..e2495e3221 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -3,6 +3,8 @@ import logging from pytest_httpserver import HTTPServer +from bbot.core.helpers.interactsh import server_list as interactsh_servers + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_sessionfinish(session, exitstatus): @@ -21,7 +23,7 @@ def pytest_sessionfinish(session, exitstatus): @pytest.fixture def non_mocked_hosts() -> list: - return ["127.0.0.1"] + return ["127.0.0.1"] + interactsh_servers @pytest.fixture diff --git a/bbot/test/test_step_1/module_tests/base.py b/bbot/test/test_step_1/module_tests/base.py index 41294b2c6c..99a069a395 100644 --- a/bbot/test/test_step_1/module_tests/base.py +++ b/bbot/test/test_step_1/module_tests/base.py @@ -39,16 +39,6 @@ def to_text(self): return str(self) -class TestClass: - @pytest_asyncio.fixture - async def my_fixture(self, bbot_httpserver): - yield bbot_httpserver - - @pytest.mark.asyncio - async def test_asdf(self, my_fixture): - log.critical(my_fixture) - - class ModuleTestBase: targets = ["blacklanternsecurity.com"] scan_name = None diff --git a/bbot/test/test_step_1/module_tests/test_module_aggregate.py b/bbot/test/test_step_1/module_tests/test_module_aggregate.py index b3d72c57e9..1049fb2a22 100644 --- a/bbot/test/test_step_1/module_tests/test_module_aggregate.py +++ b/bbot/test/test_step_1/module_tests/test_module_aggregate.py @@ -5,7 +5,6 @@ class TestAggregate(ModuleTestBase): config_overrides = {"dns_resolution": True} def check(self, module_test, events): - module_test.log.critical(events) filename = next(module_test.scan.home.glob("scan-stats-table*.txt")) with open(filename) as f: assert "| A " in f.read() diff --git a/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py index 4b2617a6bb..a77630ecd9 100644 --- a/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_1/module_tests/test_module_dnscommonsrv.py @@ -2,18 +2,23 @@ class TestDNSCommonSRV(ModuleTestBase): + targets = ["blacklanternsecurity.notreal"] + def setup_after_prep(self, module_test): old_resolve_fn = module_test.scan.helpers.dns.resolve async def resolve(query, **kwargs): - if query == "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" and kwargs.get("type", "").upper() == "SRV": - return {"asdf.blacklanternsecurity.com"} + if ( + query == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" + and kwargs.get("type", "").upper() == "SRV" + ): + return {"asdf.blacklanternsecurity.notreal"} return await old_resolve_fn(query, **kwargs) module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "resolve", resolve) def check(self, module_test, events): assert any( - e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" for e in events + e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" for e in events ), "Failed to detect subdomain" - assert not any(e.data == "_ldap._tcp.dc._msdcs.blacklanternsecurity.com" for e in events), "False positive" + assert not any(e.data == "_ldap._tcp.dc._msdcs.blacklanternsecurity.notreal" for e in events), "False positive" diff --git a/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py index 40f450a974..212fce0d6b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py +++ b/bbot/test/test_step_1/module_tests/test_module_dnszonetransfer.py @@ -15,7 +15,7 @@ def setup_after_prep(self, module_test): async def _resolve_hostname(query, **kwargs): if query == "blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() == "NS": return [module_test.mock_record("ns01.blacklanternsecurity.fakedomain", "NS")], [] - if query == "ns01.blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() in "A": + if query == "ns01.blacklanternsecurity.fakedomain" and kwargs.get("rdtype", "").upper() == "A": return [module_test.mock_record("127.0.0.1", "A")], [] return await old_resolve_fn(query, **kwargs) diff --git a/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py index 2d9cb1a7ae..9e5ed6264b 100644 --- a/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py +++ b/bbot/test/test_step_1/module_tests/test_module_generic_ssrf.py @@ -2,5 +2,8 @@ class TestGeneric_SSRF(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + # PAUL TODO - pass + def check(self, module_test, events): + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_host_header.py b/bbot/test/test_step_1/module_tests/test_module_host_header.py index 75f24378e7..43d6bc6a0d 100644 --- a/bbot/test/test_step_1/module_tests/test_module_host_header.py +++ b/bbot/test/test_step_1/module_tests/test_module_host_header.py @@ -2,5 +2,8 @@ class TestHost_Header(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + # PAUL TODO - pass + def check(self, module_test, events): + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py b/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py index 6463c2cfaf..be3e234e29 100644 --- a/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py +++ b/bbot/test/test_step_1/module_tests/test_module_ipneighbor.py @@ -3,18 +3,25 @@ class TestIPNeighbor(ModuleTestBase): targets = ["127.0.0.15", "www.bls.notreal"] - config_overrides = {"scope_report_distance": 1, "dns_resolution": True} + config_overrides = {"scope_report_distance": 1, "dns_resolution": True, "scope_dns_search_distance": 2} def setup_after_prep(self, module_test): - old_resolve_fn = module_test.scan.helpers.dns.resolve + old_resolve_ip = module_test.scan.helpers.dns._resolve_ip + old_resolve_hostname = module_test.scan.helpers.dns._resolve_hostname - async def resolve(query, **kwargs): - module_test.log.critical(f"{query}: {kwargs}") - if query == "127.0.0.3" and kwargs.get("type", "").upper() == "PTR": - return {"www.bls.notreal"} - return await old_resolve_fn(query, **kwargs) + async def _resolve_ip(query, **kwargs): + if query == "127.0.0.3": + return [module_test.mock_record("asdf.www.bls.notreal", "PTR")], [] + return await old_resolve_ip(query, **kwargs) - module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "resolve", resolve) + async def _resolve_hostname(query, **kwargs): + if query == "asdf.www.bls.notreal" and kwargs.get("rdtype", "") == "A": + return [module_test.mock_record("127.0.0.3", "A")], [] + return await old_resolve_hostname(query, **kwargs) + + module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "_resolve_ip", _resolve_ip) + module_test.monkeypatch.setattr(module_test.scan.helpers.dns, "_resolve_hostname", _resolve_hostname) def check(self, module_test, events): assert any(e.data == "127.0.0.3" for e in events) + assert not any(e.data == "127.0.0.4" for e in events) diff --git a/bbot/test/test_step_1/module_tests/test_module_smuggler.py b/bbot/test/test_step_1/module_tests/test_module_smuggler.py index e69de29bb2..955fcffde4 100644 --- a/bbot/test/test_step_1/module_tests/test_module_smuggler.py +++ b/bbot/test/test_step_1/module_tests/test_module_smuggler.py @@ -0,0 +1,9 @@ +from .base import ModuleTestBase + + +class TestSmuggler(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + + # PAUL TODO + def check(self, module_test, events): + pass diff --git a/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py b/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py index c0c1006279..7349fb187c 100644 --- a/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py +++ b/bbot/test/test_step_1/module_tests/test_module_subdomain_hijack.py @@ -30,7 +30,6 @@ def setup_after_prep(self, module_test): fingerprint = next(iter(fingerprints)) rand_string = module_test.scan.helpers.rand_string(length=15, digits=False) self.rand_subdomain = f"{rand_string}.{next(iter(fingerprint.domains))}" - module_test.log.critical(self.rand_subdomain) respond_args = {"response_data": f''} module_test.set_expect_requests(respond_args=respond_args) diff --git a/bbot/test/test_step_1/module_tests/test_module_vhost.py b/bbot/test/test_step_1/module_tests/test_module_vhost.py index e69de29bb2..bfd8037196 100644 --- a/bbot/test/test_step_1/module_tests/test_module_vhost.py +++ b/bbot/test/test_step_1/module_tests/test_module_vhost.py @@ -0,0 +1,65 @@ +from .base import ModuleTestBase, tempwordlist + + +class TestVhost(ModuleTestBase): + targets = ["http://localhost:8888", "secret.localhost"] + modules_overrides = ["httpx", "vhost"] + test_wordlist = ["11111111", "admin", "cloud", "junkword1", "zzzjunkword2"] + config_overrides = { + "modules": { + "vhost": { + "wordlist": tempwordlist(test_wordlist), + } + } + } + + def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "admin.localhost:8888"}} + respond_args = {"response_data": "Alive vhost admin"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "cloud.localhost:8888"}} + respond_args = {"response_data": "Alive vhost cloud"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "q-cloud.localhost:8888"}} + respond_args = {"response_data": "Alive vhost q-cloud"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "secret.localhost:8888"}} + respond_args = {"response_data": "Alive vhost secret"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/", "headers": {"Host": "host.docker.internal"}} + respond_args = {"response_data": "Alive vhost host.docker.internal"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "alive"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + basic_detection = False + mutaton_of_detected = False + basehost_mutation = False + special_vhost_list = False + wordcloud_detection = False + + for e in events: + if e.type == "VHOST": + if e.data["vhost"] == "admin": + basic_detection = True + if e.data["vhost"] == "cloud": + mutaton_of_detected = True + if e.data["vhost"] == "q-cloud": + basehost_mutation = True + if e.data["vhost"] == "host.docker.internal": + special_vhost_list = True + if e.data["vhost"] == "secret": + wordcloud_detection = True + + assert basic_detection + assert mutaton_of_detected + assert basehost_mutation + assert special_vhost_list + assert wordcloud_detection diff --git a/bbot/test/test_step_1/module_tests/test_module_wayback.py b/bbot/test/test_step_1/module_tests/test_module_wayback.py index e69de29bb2..3395e0052a 100644 --- a/bbot/test/test_step_1/module_tests/test_module_wayback.py +++ b/bbot/test/test_step_1/module_tests/test_module_wayback.py @@ -0,0 +1,12 @@ +from .base import ModuleTestBase + + +class TestWayback(ModuleTestBase): + def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original", + json=[["original"], ["http://asdf.blacklanternsecurity.com"]], + ) + + def check(self, module_test, events): + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" diff --git a/bbot/test/test_step_1/module_tests/test_module_web_report.py b/bbot/test/test_step_1/module_tests/test_module_web_report.py index e69de29bb2..14b911c19f 100644 --- a/bbot/test/test_step_1/module_tests/test_module_web_report.py +++ b/bbot/test/test_step_1/module_tests/test_module_web_report.py @@ -0,0 +1,65 @@ +from .base import ModuleTestBase + + +class TestWebReport(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "wappalyzer", "badsecrets", "web_report", "secretsdb"] + + def setup_before_prep(self, module_test): + # secretsdb --> FINDING + module_test.httpx_mock.add_response( + url="https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml", + text="""patterns: +- pattern: + confidence: 99 + name: Asymmetric Private Key + regex: '-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'""", + ) + # wappalyzer --> TECHNOLOGY + # badsecrets --> VULNERABILITY + respond_args = {"response_data": web_body} + module_test.set_expect_requests(respond_args=respond_args) + module_test.httpx_mock.assert_all_responses_were_requested = False + + def check(self, module_test, events): + report_file = module_test.scan.home / "web_report.html" + with open(report_file) as f: + report_content = f.read() + assert "
  • [HIGH] Known Secret Found" in report_content + assert ( + """

    URL

    +