From b8cfdd21898b7fb21c1273625e6e97edd1c30814 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 7 Oct 2023 10:45:11 -0400 Subject: [PATCH] steady work on scope shepherding --- bbot/core/event/base.py | 90 +-- bbot/core/helpers/dns.py | 52 +- bbot/modules/base.py | 68 +- bbot/modules/httpx.py | 1 + bbot/modules/internal/speculate.py | 3 +- bbot/scanner/manager.py | 161 ++-- bbot/scanner/target.py | 2 +- bbot/test/bbot_fixtures.py | 2 +- bbot/test/conftest.py | 4 +- bbot/test/test_step_1/test_events.py | 10 +- bbot/test/test_step_1/test_manager.py | 2 +- .../test_manager_scope_shepherding.py | 687 ++++++++++++++++++ bbot/test/test_step_1/test_modules_basic.py | 12 +- pyproject.toml | 3 +- 14 files changed, 909 insertions(+), 188 deletions(-) create mode 100644 bbot/test/test_step_1/test_manager_scope_shepherding.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index a198a89bcf..a4da3bae5a 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -143,9 +143,6 @@ def __init__( self._module_priority = None self._resolved_hosts = set() - self._made_internal = False - # whether to force-output this event to the graph - self._graph_important = False # keep track of whether this event has been recorded by the scan self._stats_recorded = False @@ -292,7 +289,9 @@ def remove_tag(self, tag): @property def always_emit(self): - return self._always_emit or any(t in self.tags for t in self._always_emit_tags) + always_emit_tags = any(t in self.tags for t in self._always_emit_tags) + no_host_information = not bool(self.host) + return self._always_emit or always_emit_tags or no_host_information @property def id(self): @@ -320,6 +319,10 @@ def scope_distance(self, scope_distance): The method will automatically update the relevant 'distance-' tags associated with the event. """ if scope_distance >= 0: + if scope_distance == 0: + self.add_tag("in-scope") + else: + self.remove_tag("in-scope") new_scope_distance = None # ensure scope distance does not increase (only allow setting to smaller values) if self.scope_distance == -1: @@ -332,6 +335,10 @@ def scope_distance(self, scope_distance): if t.startswith("distance-"): self.remove_tag(t) self.add_tag(f"distance-{new_scope_distance}") + # apply recursively to parent events + source_scope_distance = getattr(self.source, "scope_distance", -1) + if source_scope_distance >= 0 and self != self.source: + self.source.scope_distance = scope_distance + 1 @property def source(self): @@ -411,75 +418,8 @@ def make_internal(self): The purpose of internal events is to enable speculative/explorative discovery without cluttering the console with irrelevant or uninteresting events. """ - if not self._made_internal: - self._internal = True - self.add_tag("internal") - self._made_internal = True - - def unmake_internal(self, set_scope_distance=None, graph_important=False): - """ - Reverts the event from being internal, optionally forcing it to be included in output and setting its scope distance. - - Removes the 'internal' tag, resets the `_internal` attribute, and adjusts scope distance if specified. - Optionally, forces the event to be included in the output. Also, if any source events are internal, they - are also reverted recursively. - - This typically happens in `ScanManager._emit_event()` if the event is determined to be interesting. - - Parameters: - set_scope_distance (int, optional): If specified, sets the scope distance to this value. - graph_important (bool or str, optional): If True, forces the event to be included in output. - If set to "trail_only", only its source events are modified. - - Returns: - list: A list of source events that were also reverted from being internal. - """ - source_trail = [] - self.remove_tag("internal") - if self._made_internal: - if set_scope_distance is not None: - self.scope_distance = set_scope_distance - self._internal = False - self._made_internal = False - if graph_important is True: - self._graph_important = True - if graph_important == "trail_only": - graph_important = True - - # if our source event is internal, unmake it too - if getattr(self.source, "_internal", False): - source_scope_distance = None - if set_scope_distance is not None: - source_scope_distance = set_scope_distance + 1 - source_trail += self.source.unmake_internal( - set_scope_distance=source_scope_distance, graph_important=graph_important - ) - source_trail.append(self.source) - - return source_trail - - def set_scope_distance(self, d=0): - """ - Sets the scope distance for the event and its parent events, while considering module-specific scoping rules. - - Unmakes the event internal if needed and adjusts its scope distance. If the distance is set to 0, - adds the 'in-scope' tag to the event. Takes into account module-specific scoping preferences unless - the event type is "DNS_NAME". - - Parameters: - d (int): The scope distance to set for this event. - - Returns: - list: A list of parent events whose scope distance was also set. - """ - source_trail = [] - # keep the event internal if the module requests so, unless it's a DNS_NAME - if getattr(self.module, "_scope_shepherding", True) or self.type in ("DNS_NAME",): - source_trail = self.unmake_internal(set_scope_distance=d, graph_important="trail_only") - self.scope_distance = d - if d == 0: - self.add_tag("in-scope") - return source_trail + self._internal = True + self.add_tag("internal") def _host(self): return "" @@ -747,7 +687,7 @@ def __hash__(self): def __str__(self): max_event_len = 80 d = str(self.data) - return f'{self.type}("{d[:max_event_len]}{("..." if len(d) > max_event_len else "")}", module={self.module}, tags={self.tags})' + return f'{self.type}("{d[:max_event_len]}{("..." if len(d) > max_event_len else "")}", module={self.module}, tags={self.tags} graph_important={self._graph_important})' def __repr__(self): return str(self) @@ -1225,7 +1165,7 @@ def make_event( data.module = module if source is not None: data.source = source - if internal == True and not data._made_internal: + if internal == True: data.make_internal() event_type = data.type return data diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 8608536614..58dbce3e20 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -1,3 +1,4 @@ +import dns import time import asyncio import logging @@ -89,7 +90,7 @@ def __init__(self, parent_helper): self.timeout = self.parent_helper.config.get("dns_timeout", 5) self.retries = self.parent_helper.config.get("dns_retries", 1) self.abort_threshold = self.parent_helper.config.get("dns_abort_threshold", 50) - self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 4) + self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 5) self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout self._resolver_list = None @@ -132,6 +133,10 @@ def __init__(self, parent_helper): self._event_cache = self.parent_helper.CacheDict(max_size=10000) self._event_cache_locks = NamedLock() + # for mocking DNS queries + self._orig_resolve_raw = None + self._mock_table = {} + # copy the system's current resolvers to a text file for tool use self.system_resolvers = dns.resolver.Resolver().nameservers self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False) @@ -220,13 +225,7 @@ async def resolve_raw(self, query, **kwargs): kwargs.pop("rdtype", None) if "type" in kwargs: t = kwargs.pop("type") - if isinstance(t, str): - if t.strip().lower() in ("any", "all", "*"): - types = self.all_rdtypes - else: - types = [t.strip().upper()] - elif any([isinstance(t, x) for x in (list, tuple)]): - types = [str(_).strip().upper() for _ in t] + types = self._parse_rdtype(t, default=types) for t in types: r, e = await self._resolve_hostname(query, rdtype=t, **kwargs) if r: @@ -500,7 +499,7 @@ async def resolve_event(self, event, minimal=False): event_blacklisted = False try: - if not event.host or event.type in ("IP_RANGE",): + if (not event.host) or (event.type in ("IP_RANGE",)): return event_tags, event_whitelisted, event_blacklisted, dns_children # lock to ensure resolution of the same host doesn't start while we're working here @@ -1016,6 +1015,16 @@ async def _connectivity_check(self, interval=5): self._errors.clear() return False + def _parse_rdtype(self, t, default=None): + if isinstance(t, str): + if t.strip().lower() in ("any", "all", "*"): + return self.all_rdtypes + else: + return [t.strip().upper()] + elif any([isinstance(t, x) for x in (list, tuple)]): + return [str(_).strip().upper() for _ in t] + return default + def debug(self, *args, **kwargs): if self._debug: log.debug(*args, **kwargs) @@ -1027,3 +1036,28 @@ def _get_dummy_module(self, name): dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS") self._dummy_modules[name] = dummy_module return dummy_module + + def mock_dns(self, dns_dict): + if self._orig_resolve_raw is None: + self._orig_resolve_raw = self.resolve_raw + + async def mock_resolve_raw(query, **kwargs): + results = [] + errors = [] + types = self._parse_rdtype(kwargs.get("type", ["A", "AAAA"])) + for t in types: + with suppress(KeyError): + results += self._mock_table[(query, t)] + return results, errors + + for (query, rdtype), answers in dns_dict.items(): + if isinstance(answers, str): + answers = [answers] + for answer in answers: + rdata = dns.rdata.from_text("IN", rdtype, answer) + try: + self._mock_table[(query, rdtype)].append((rdtype, rdata)) + except KeyError: + self._mock_table[(query, rdtype)] = [(rdtype, [rdata])] + + self.resolve_raw = mock_resolve_raw diff --git a/bbot/modules/base.py b/bbot/modules/base.py index ab24e25971..00e5b96ca6 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -125,6 +125,10 @@ def __init__(self, scan): self._log = None self._incoming_event_queue = None self._outgoing_event_queue = None + # track incoming events to prevent unwanted duplicates + self._incoming_dup_tracker = set() + # track events that are critical to the graph + self._graph_important_tracker = set() # seconds since we've submitted a batch self._last_submitted_batch = None # additional callbacks to be executed alongside self.cleanup() @@ -671,6 +675,30 @@ def _event_precheck(self, event): return True, "precheck succeeded" async def _event_postcheck(self, event): + """ + A simple wrapper for dup tracking and preserving event chains for graph modules + """ + acceptable, reason = await self.__event_postcheck(event) + if acceptable: + is_incoming_duplicate = self.is_incoming_duplicate(event, add=True) + if is_incoming_duplicate and not self.accept_dupes: + if not self._graph_important(event): + return False, f"module has already seen {event}" + + if self._preserve_graph: + s = event + while 1: + s = s.source + if s is None or s == self.scan.root_event or s == event: + break + if not self.is_incoming_duplicate(s, add=True): + self._graph_important_tracker.add(hash(event)) + self.critical(f"queueing {event}") + await self.queue_event(s, precheck=False) + + return acceptable, reason + + async def __event_postcheck(self, event): """ Post-checks an event to determine if it should be accepted by the module for handling. @@ -683,14 +711,6 @@ async def _event_postcheck(self, event): Returns: tuple: A tuple (bool, str) where the bool indicates if the event should be accepted, and the str gives the reason. - Examples: - >>> async def custom_filter(event): - ... if event.data not in ["evilcorp.com"]: - ... return False, "it's not on the cool list" - ... - >>> self.filter_event = custom_filter - >>> result, reason = await self._event_postcheck(event) - Notes: - Override the `filter_event` method for custom filtering logic. - This method also maintains host-based tracking when the `per_host_only` flag is set. @@ -709,6 +729,8 @@ async def _event_postcheck(self, event): # check scope distance filter_result, reason = self._scope_distance_check(event) if not filter_result: + if self._is_graph_important(event): + return True, f"{reason}, but exception was made because it is graph important" return filter_result, reason # custom filtering @@ -765,7 +787,7 @@ async def _cleanup(self): async with self.scan._acatch(context), self._task_counter.count(context): await self.helpers.execute_sync_or_async(callback) - async def queue_event(self, event): + async def queue_event(self, event, precheck=True): """ Asynchronously queues an incoming event to the module's event queue for further processing. @@ -788,7 +810,9 @@ async def queue_event(self, event): if self.incoming_event_queue is False: self.debug(f"Not in an acceptable state to queue incoming event") return - acceptable, reason = self._event_precheck(event) + acceptable, reason = True, "no precheck was performed" + if precheck: + acceptable, reason = self._event_precheck(event) if not acceptable: if reason and reason != "its type is not in watched_events": self.debug(f"Not accepting {event} because {reason}") @@ -871,6 +895,30 @@ def set_error_state(self, message=None, clear_outgoing_queue=False): while 1: self.outgoing_event_queue.get_nowait() + def is_incoming_duplicate(self, event, add=False): + event_hash = self._incoming_dedup_hash(event) + is_dup = event_hash in self._incoming_dup_tracker + if add: + self._incoming_dup_tracker.add(event_hash) + if self._is_graph_important(event): + return False + return is_dup + + def _is_graph_important(self, event): + return self._preserve_graph and hash(event) in self._graph_important_tracker + + def _incoming_dedup_hash(self, event): + """ + Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False. + """ + return hash(event) + + def _outgoing_dedup_hash(self, event): + """ + Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True. + """ + return hash(event) + def get_per_host_hash(self, event): """ Computes a per-host hash value for a given event. This method may be optionally overridden in subclasses. diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index ef77668db0..b1aab8ee0f 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -110,6 +110,7 @@ async def handle_batch(self, *events): if proxy: command += ["-http-proxy", proxy] async for line in self.helpers.run_live(command, input=list(stdin), stderr=subprocess.DEVNULL): + self.critical(line) try: j = json.loads(line) except json.decoder.JSONDecodeError: diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 4aa3bb6166..c465a673ad 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -1,7 +1,6 @@ import random import ipaddress -from bbot.core.helpers.misc import parse_port_string from bbot.modules.internal.base import BaseInternalModule @@ -43,7 +42,7 @@ async def setup(self): port_string = self.config.get("ports", "80,443") try: - self.ports = parse_port_string(port_string) + self.ports = self.helpers.parse_port_string(port_string) except ValueError as e: self.warning(f"Error parsing ports: {e}") return False diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 38a247afe1..2d2bac754f 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -39,13 +39,21 @@ def __init__(self, scan): self.scan = scan - self.incoming_event_queue = asyncio.PriorityQueue() + # TODO: consider reworking modules' dedupe policy (accept_dupes) + # by creating a function that decides the criteria for what is + # considered to be a duplicate (by default this would be a simple + # hash(event)), but allowing each module to override it if needed. + # If a module used the default function, its dedupe could be done + # at the manager level to save memory. If not, it would be done by the scan. - # tracks duplicate events on a global basis - self.events_distributed = set() - # tracks duplicate events on a per-module basis + self.incoming_event_queue = asyncio.PriorityQueue() + # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules) + self.incoming_dup_tracker = set() + # track outgoing duplicates (for `accept_dupes` attribute of modules) + self.outgoing_dup_tracker = set() + # tracks duplicate events self.events_accepted = set() - # tracks duplicate events on a graph basis + # tracks duplicate events for graph purposes # (allows certain duplicates in order to maintain a complete graph) self.events_accepted_graph = set() self.dns_resolution = self.scan.config.get("dns_resolution", False) @@ -82,15 +90,19 @@ async def emit_event(self, event, *args, **kwargs): bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com")) """ async with self._task_counter.count(f"emit_event({event})"): + # "quick" queues the event immediately + # This is used by speculate + quick = kwargs.pop("quick", False) + # skip event if it fails precheck - if not self._event_precheck(event): - event._resolved.set() - return + if event.type != "DNS_NAME": + acceptable = self._event_precheck(event) + if not acceptable: + event._resolved.set() + return log.debug(f'Module "{event.module}" raised {event}') - # "quick" queues the event immediately - quick = kwargs.pop("quick", False) if quick: log.debug(f'Module "{event.module}" raised {event}') event._resolved.set() @@ -100,11 +112,15 @@ async def emit_event(self, event, *args, **kwargs): await self.distribute_event(event, *args, **kwargs) else: async with self.scan._acatch(context=self._emit_event, finally_callback=event._resolved.set): - await self._emit_event(event, *args, **kwargs) + await self._emit_event( + event, + *args, + **kwargs, + ) def _event_precheck(self, event): """ - Check an event previous to its DNS resolution etc. to see if we can save on performance by skipping it + Check an event to see if we can skip it to save on performance """ if event._dummy: log.warning(f"Cannot emit dummy event: {event}") @@ -112,8 +128,8 @@ def _event_precheck(self, event): if event == event.get_source(): log.debug(f"Skipping event with self as source: {event}") return False - if self.is_duplicate_event(event) and self.is_graph_duplicate(event): - log.debug(f"Skipping {event} because it is a duplicate") + if self.is_incoming_duplicate(event, add=True): + log.debug(f"Skipping event because it was already emitted by its module: {event}") return False return True @@ -207,7 +223,7 @@ async def _emit_event(self, event, **kwargs): if event_blacklisted_dns: reason = "DNS associations" log.debug(f"Omitting due to blacklisted {reason}: {event}") - distribute_event = False + return # DNS_NAME --> DNS_NAME_UNRESOLVED if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags: @@ -219,18 +235,14 @@ async def _emit_event(self, event, **kwargs): # Scope shepherding # here is where we make sure in-scope events are set to their proper scope distance - if event.host: - if event_whitelisted: - log.debug(f"Making {event} in-scope") - source_trail = event.set_scope_distance(0) - # force re-emit internal source events - for s in source_trail: - self.queue_event(s) - elif event.scope_distance > self.scan.scope_report_distance: - log.debug( - f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" - ) - event.make_internal() + if event.host and event_whitelisted: + log.critical(f"Making {event} in-scope") + event.scope_distance = 0 + elif (not event.always_emit) and event.scope_distance > self.scan.scope_report_distance: + log.critical( + f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})" + ) + event.make_internal() # check for wildcards if event.scope_distance <= self.scan.scope_search_distance: @@ -238,10 +250,11 @@ async def _emit_event(self, event, **kwargs): if not self.scan.helpers.is_ip_type(event.host): await self.scan.helpers.dns.handle_wildcard_event(event, dns_children) - # We do this again in case event.data changed during handle_wildcard_event() - if event.type == "DNS_NAME" and not self._event_precheck(event): - log.debug(f"Omitting due to failed precheck: {event}") - distribute_event = False + # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event() + if event.type == "DNS_NAME": + acceptable = self._event_precheck(event) + if not acceptable: + return # now that the event is properly tagged, we can finally make decisions about it abort_result = False @@ -256,24 +269,12 @@ async def _emit_event(self, event, **kwargs): log.debug(msg) return - event_is_duplicate = self.is_duplicate_event(event, add=True) - # event_is_graph_duplicate = self.is_graph_duplicate(event, add=True) - # run success callback before distributing event (so it can add tags, etc.) if distribute_event: if callable(on_success_callback): async with self.scan._acatch(context=on_success_callback): await self.scan.helpers.execute_sync_or_async(on_success_callback, event) - # Force-emit certain events - if not event.host or (event.always_emit and not event_is_duplicate): - log.debug( - f"Force-emitting {event} (host:{event.host}, always_emit={event.always_emit}, is_duplicate={event_is_duplicate})" - ) - source_trail = event.unmake_internal(graph_important=True) - for s in source_trail: - self.queue_event(s) - await self.distribute_event(event) event_distributed = True @@ -286,7 +287,9 @@ async def _emit_event(self, event, **kwargs): ): source_module = self.scan.helpers._make_dummy_module("host", _type="internal") source_module._priority = 4 - source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event) + source_event = self.scan.make_event( + event.host, "DNS_NAME", module=source_module, source=event, internal=True + ) # only emit the event if it's not already in the parent chain if source_event is not None and source_event not in source_event.get_sources(): source_event.scope_distance = event.scope_distance @@ -296,13 +299,13 @@ async def _emit_event(self, event, **kwargs): ### Emit DNS children ### if self.dns_resolution: - emit_children = -1 < event.scope_distance < self.scan.scope_dns_search_distance - if emit_children: - # only emit DNS children once for each unique host - host_hash = hash(str(event.host)) - if host_hash in self.events_accepted: - emit_children = False - self.events_accepted.add(host_hash) + emit_children = True + in_dns_scope = -1 < event.scope_distance < self.scan.scope_dns_search_distance + # only emit DNS children once for each unique host + host_hash = hash(str(event.host)) + if host_hash in self.outgoing_dup_tracker: + emit_children = False + self.outgoing_dup_tracker.add(host_hash) if emit_children: dns_child_events = [] @@ -315,7 +318,9 @@ async def _emit_event(self, event, **kwargs): child_event = self.scan.make_event( record, "DNS_NAME", module=module, source=source_event ) - dns_child_events.append(child_event) + host_hash = hash(str(child_event.host)) + if in_dns_scope or self.scan.in_scope(child_event): + dns_child_events.append(child_event) except ValidationError as e: log.warning( f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}' @@ -347,28 +352,35 @@ def hash_event_graph(self, event): else: return hash((event, str(event.module))) - def is_duplicate_event(self, event, add=False): + def is_incoming_duplicate(self, event, add=False): """ - Calculate whether an event is a duplicate on a per-module basis + Calculate whether an event is a duplicate in the context of the module that emitted it + This will return True if the event's parent module has raised the event before. """ - event_hash = hash(event) - suppress_dupes = getattr(event.module, "suppress_dupes", True) - duplicate_event = suppress_dupes and event_hash in self.events_accepted + try: + event_hash = event.module._outgoing_dedup_hash(event) + except AttributeError: + event_hash = hash(event) + is_dup = event_hash in self.incoming_dup_tracker if add: - self.events_accepted.add(event_hash) - return duplicate_event + self.incoming_dup_tracker.add(event_hash) + suppress_dupes = getattr(event.module, "suppress_dupes", True) + if suppress_dupes and is_dup: + return True + return False - def is_graph_duplicate(self, event, add=False): + def is_outgoing_duplicate(self, event, add=False): """ - Calculate whether an event is a duplicate for graphing purposes + Calculate whether an event is a duplicate in the context of the whole scan, + This will return True if the same event (irregardless of its source module) has been emitted before. + + TODO: Allow modules to use this for custom deduplication such as on a per-host or per-domain basis. """ - if event._graph_important: - return False - event_hash = self.hash_event_graph(event) - duplicate_event = event_hash in self.events_accepted_graph + event_hash = hash(event) + is_dup = event_hash in self.outgoing_dup_tracker if add: - self.events_accepted_graph.add(event_hash) - return duplicate_event + self.outgoing_dup_tracker.add(event_hash) + return is_dup async def distribute_event(self, *args, **kwargs): """ @@ -376,19 +388,18 @@ async def distribute_event(self, *args, **kwargs): """ async with self.scan._acatch(context=self.distribute_event): event = self.scan.make_event(*args, **kwargs) - - event_hash = hash(event) - dup = event_hash in self.events_distributed - if dup: + is_outgoing_duplicate = self.is_outgoing_duplicate(event) + if is_outgoing_duplicate: self.scan.verbose(f"{event.module}: Duplicate event: {event}") - else: - self.events_distributed.add(event_hash) # absorb event into the word cloud if it's in scope - if not dup and -1 < event.scope_distance < 1: + if not is_outgoing_duplicate and -1 < event.scope_distance < 1: self.scan.word_cloud.absorb_event(event) for mod in self.scan.modules.values(): - if (not dup) or (mod.accept_dupes) or (mod._preserve_graph and event._graph_important): + acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes + if acceptable_dup: await mod.queue_event(event) + else: + log.critical(f"Not distributing {event} to {mod}") async def _worker_loop(self): try: diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index f733f82956..e22ed97246 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -128,7 +128,7 @@ def add_target(self, t): t, source=self.scan.root_event, module=self._dummy_module, tags=["target"] ) if self.make_in_scope: - event.set_scope_distance(0) + event.scope_distance = 0 try: self._events[event.host].add(event) except KeyError: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 2c74fe1906..a08ef3ecef 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -210,7 +210,7 @@ class bbot_events: ] for e in bbot_events.all: - e.set_scope_distance(0) + e.scope_distance = 0 return bbot_events diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 67e7515ff7..a4c9827057 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,5 +1,5 @@ import ssl -import shutil +import shutil # noqa import pytest import logging from pathlib import Path @@ -18,7 +18,7 @@ def pytest_sessionfinish(session, exitstatus): logger.removeHandler(handler) # Wipe out BBOT home dir - shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) + # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 842b91f9cb..a16b5a9232 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -142,7 +142,7 @@ async def test_events(events, scan, helpers, bbot_config): # scope distance event1 = scan.make_event("1.2.3.4", dummy=True) assert event1._scope_distance == -1 - event1.set_scope_distance(0) + event1.scope_distance = 0 assert event1._scope_distance == 0 event2 = scan.make_event("2.3.4.5", source=event1) assert event2._scope_distance == 1 @@ -157,14 +157,14 @@ async def test_events(events, scan, helpers, bbot_config): root_event = scan.make_event("0.0.0.0", dummy=True) internal_event1 = scan.make_event("1.2.3.4", source=root_event, internal=True) assert internal_event1._internal == True - assert internal_event1._made_internal == True - internal_event1.set_scope_distance(0) + assert "internal" in internal_event1.tags + internal_event1.scope_distance = 0 assert internal_event1._internal == False - assert internal_event1._made_internal == False + assert "internal" not in internal_event1.tags internal_event2 = scan.make_event("2.3.4.5", source=internal_event1, internal=True) internal_event3 = scan.make_event("3.4.5.6", source=internal_event2, internal=True) internal_event4 = scan.make_event("4.5.6.7", source=internal_event3) - source_trail = internal_event4.set_scope_distance(0) + source_trail = internal_event4.scope_distance = 0 assert internal_event4._internal == False assert internal_event3._internal == False assert internal_event2._internal == False diff --git a/bbot/test/test_step_1/test_manager.py b/bbot/test/test_step_1/test_manager.py index 16e6db7f56..7a17dda68d 100644 --- a/bbot/test/test_step_1/test_manager.py +++ b/bbot/test/test_step_1/test_manager.py @@ -72,7 +72,7 @@ class DummyModule3: googledns = scan1.make_event("8.8.8.8", source=scan1.root_event) googledns.module = DummyModule2() googledns.source = "asdf" - googledns.set_scope_distance(0) + googledns.scope_distance = 0 manager.queue_event = event_children_append await manager._emit_event(googledns) assert len(event_children) > 0 diff --git a/bbot/test/test_step_1/test_manager_scope_shepherding.py b/bbot/test/test_step_1/test_manager_scope_shepherding.py new file mode 100644 index 0000000000..a3fb7df414 --- /dev/null +++ b/bbot/test/test_step_1/test_manager_scope_shepherding.py @@ -0,0 +1,687 @@ +from ..bbot_fixtures import * # noqa: F401 + +from pytest_httpserver import HTTPServer + + +@pytest.fixture +def bbot_other_httpserver(): + server = HTTPServer(host="127.0.0.77", port=8888) + server.start() + + yield server + + server.clear() + if server.is_running(): + server.stop() + + server.check_assertions() + server.clear() + + +@pytest.fixture +def bbot_other_httpserver2(): + server = HTTPServer(host="127.0.0.88", port=8888) + server.start() + + yield server + + server.clear() + if server.is_running(): + server.stop() + + server.check_assertions() + server.clear() + + +@pytest.fixture +def bbot_other_httpserver3(): + server = HTTPServer(host="127.0.0.111", port=8888) + server.start() + + yield server + + server.clear() + if server.is_running(): + server.stop() + + server.check_assertions() + server.clear() + + +@pytest.fixture +def bbot_other_httpserver4(): + server = HTTPServer(host="127.0.0.222", port=8888) + server.start() + + yield server + + server.clear() + if server.is_running(): + server.stop() + + server.check_assertions() + server.clear() + + + + +@pytest.mark.asyncio +async def test_manager_scope_shepherding(bbot_config, bbot_scanner, bbot_httpserver, bbot_other_httpserver, bbot_other_httpserver2, bbot_other_httpserver3, bbot_other_httpserver4): + from bbot.modules.base import BaseModule + from bbot.modules.output.base import BaseOutputModule + + class DummyModule(BaseModule): + _name = "dummymodule" + watched_events = ["*"] + scope_distance_modifier = 10 + accept_dupes = True + + async def setup(self): + self.events = [] + return True + + async def handle_event(self, event): + self.events.append(event) + + class DummyModuleNoDupes(DummyModule): + accept_dupes = False + + class DummyGraphModule(DummyModule): + _name = "dummygraphmodule" + watched_events = ["*"] + scope_distance_modifier = 0 + accept_dupes = True + _preserve_graph = True + + class DummyGraphOutputModule(BaseOutputModule): + _name = "dummygraphoutputmodule" + watched_events = ["*"] + _preserve_graph = True + + async def setup(self): + self.events = [] + return True + + async def handle_event(self, event): + self.events.append(event) + + async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs): + merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(_config)) + scan = bbot_scanner(*args, config=merged_config, **kwargs) + dummymodule = DummyModule(scan) + dummymodulenodupes = DummyModuleNoDupes(scan) + dummygraphmodule = DummyGraphModule(scan) + dummygraphoutputmodule = DummyGraphOutputModule(scan) + scan.modules["dummymodule"] = dummymodule + scan.modules["dummymodulenodupes"] = dummymodulenodupes + scan.modules["dummygraphmodule"] = dummygraphmodule + scan.modules["dummygraphoutputmodule"] = dummygraphoutputmodule + if _dns_mock: + scan.helpers.dns.mock_dns(_dns_mock) + if scan_callback is not None: + scan_callback(scan) + return ( + [e async for e in scan.async_start()], + dummymodule.events, + dummymodulenodupes.events, + dummygraphmodule.events, + dummygraphoutputmodule.events, + ) + + dns_mock_chain = { + ("test.notreal", "A"): "127.0.0.66", + ("127.0.0.66", "PTR"): "test.notrealzies", + ("test.notrealzies", "CNAME"): "www.test.notreal", + ("www.test.notreal", "A"): "127.0.0.77", + ("127.0.0.77", "PTR"): "test2.notrealzies", + ("test2.notrealzies", "A"): "127.0.0.88", + } + + """ + + # dns search distance = 1, report distance = 0 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "test.notreal", + _config={"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0}, + _dns_mock=dns_mock_chain, + ) + + assert len(events) == 2 + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + + assert len(all_events) == 3 + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) + assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + + assert len(all_events_nodups) == 3 + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) + assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + + assert len(graph_events) == 2 + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) + assert 0 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + + assert len(graph_output_events) == 2 + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) + assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + + """ + + # dns search distance = 2, report distance = 0 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "test.notreal", + _config={"dns_resolution": True, "scope_dns_search_distance": 2, "scope_report_distance": 0}, + _dns_mock=dns_mock_chain, + ) + + for e in events: + log.critical(e) + log.critical("=" * 20) + for e in all_events: + log.critical(e) + log.critical("=" * 20) + for e in all_events_nodups: + log.critical(e) + log.critical("=" * 20) + for e in graph_events: + log.critical(e) + log.critical("=" * 20) + for e in graph_output_events: + log.critical(e) + + assert len(events) == 3 + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(all_events) == 7 + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(all_events_nodups) == 7 + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(graph_events) == 5 + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(graph_output_events) == 5 + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + return + + """ + + # dns search distance = 2, report distance = 1 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "test.notreal", + _config={"dns_resolution": True, "scope_dns_search_distance": 2, "scope_report_distance": 1}, + _dns_mock=dns_mock_chain, + ) + + assert len(events) == 5 + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(all_events) == 7 + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(all_events_nodups) == 7 + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(graph_events) == 5 + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + assert len(graph_output_events) == 7 + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 2 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + + dns_mock_chain = { + ("test.notreal", "A"): "127.0.0.66", + ("127.0.0.66", "PTR"): "test.notrealzies", + ("test.notrealzies", "A"): "127.0.0.77", + } + + class DummyVulnModule(BaseModule): + _name = "dummyvulnmodule" + watched_events = ["IP_ADDRESS"] + scope_distance_modifier = 3 + accept_dupes = True + + async def filter_event(self, event): + if event.data == "127.0.0.77": + return True + return False, "bleh" + + async def handle_event(self, event): + self.emit_event( + {"host": str(event.host), "description": "yep", "severity": "CRITICAL"}, "VULNERABILITY", source=event + ) + + def custom_setup(scan): + dummyvulnmodule = DummyVulnModule(scan) + scan.modules["dummyvulnmodule"] = dummyvulnmodule + + # dns search distance = 3, report distance = 1 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "test.notreal", + scan_callback=custom_setup, + _config={"dns_resolution": True, "scope_dns_search_distance": 3, "scope_report_distance": 1}, + _dns_mock=dns_mock_chain, + ) + + assert len(events) == 4 + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 1 == len([e for e in events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e._internal == False and e._graph_important == False]) + + assert len(all_events) == 6 + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e._internal == False and e._graph_important == False]) + + assert len(all_events_nodups) == 6 + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e._internal == False and e._graph_important == False]) + + assert len(graph_events) == 5 + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == True]) + assert 0 == len([e for e in graph_events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77"]) + + assert len(graph_output_events) == 7 + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e._internal == False and e._graph_important == False]) + assert 2 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e._internal == False and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e._internal == False and e._graph_important == False]) + """ + + bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="") + bbot_other_httpserver.expect_request(uri="/").respond_with_data(response_data="") + bbot_other_httpserver2.expect_request(uri="/").respond_with_data(response_data="") + bbot_other_httpserver3.expect_request(uri="/").respond_with_data(response_data="") + bbot_other_httpserver4.expect_request(uri="/").respond_with_data(response_data="") + + """ + + # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "127.0.0.1/31", + modules=["httpx", "excavate"], + _config={ + "scope_search_distance": 0, + "scope_dns_search_distance": 2, + "scope_report_distance": 1, + "speculate": True, + "internal_modules": {"speculate": {"ports": "8888"}}, + "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], + }, + ) + + assert len(events) == 3 + assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888"]) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"]) + assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) + + assert len(all_events) == 11 + assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + + assert len(all_events_nodups) == 11 + assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + + assert len(graph_events) == 10 + assert 1 == len([e for e in graph_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + + assert len(graph_output_events) == 5 + assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + + """ + + # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "127.0.0.1/31", + modules=["httpx", "excavate"], + output_modules=["neo4j"], + _config={ + "scope_search_distance": 0, + "scope_dns_search_distance": 2, + "scope_report_distance": 1, + "speculate": True, + "modules": {"httpx": {"in_scope_only": False}}, + "internal_modules": {"speculate": {"ports": "8888"}}, + "output_modules": {"neo4j": {"uri": "bolt://localhost:7687"}}, + "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], + }, + ) + + for e in events: + log.critical(e) + log.critical("=" * 20) + for e in all_events: + log.critical(e) + log.critical("=" * 20) + for e in all_events_nodups: + log.critical(e) + log.critical("=" * 20) + for e in graph_events: + log.critical(e) + log.critical("=" * 20) + for e in graph_output_events: + log.critical(e) + + assert len(events) == 4 + assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888"]) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"]) + assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) + + assert len(all_events) == 15 + assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(all_events_nodups) == 15 + assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(graph_events) == 10 + assert 1 == len([e for e in graph_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(graph_output_events) == 6 + assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + return + + # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 1 + events, all_events, all_events_nodups, graph_events, graph_output_events = await do_scan( + "127.0.0.1/31", + modules=["httpx", "excavate"], + _config={ + "scope_search_distance": 1, + "scope_dns_search_distance": 2, + "scope_report_distance": 1, + "speculate": True, + "modules": {"httpx": {"in_scope_only": False}}, + "internal_modules": {"speculate": {"ports": "8888"}}, + "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"], + }, + ) + + for e in events: + log.critical(e) + log.critical("=" * 20) + for e in all_events: + log.critical(e) + log.critical("=" * 20) + for e in all_events_nodups: + log.critical(e) + log.critical("=" * 20) + for e in graph_events: + log.critical(e) + log.critical("=" * 20) + for e in graph_output_events: + log.critical(e) + + assert len(events) == 4 + assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888"]) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"]) + assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) + assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"]) + assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) + assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) + + return + + assert len(all_events) == 15 + assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(all_events_nodups) == 15 + assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(graph_events) == 10 + assert 1 == len([e for e in graph_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 2 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 1 == len([e for e in graph_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) + + assert len(graph_output_events) == 5 + assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e._internal == True and e._graph_important == True]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e._internal == True and e._graph_important == False]) + assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e._internal == True and e._graph_important == True]) + assert 1 == len([e for e in graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e._internal == False and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e._internal == True and e._graph_important == False]) + assert 0 == len([e for e in graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e._internal == True and e._graph_important == False]) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 77c25a7a15..4b478e31b1 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -33,7 +33,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule): base_module = module_class(scan) localhost2 = scan.make_event("127.0.0.2", source=events.subdomain) - localhost2.set_scope_distance(0) + localhost2.scope_distance = 0 # base cases base_module._watched_events = None base_module.watched_events = ["*"] @@ -54,7 +54,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h base_module.watched_events = ["IP_ADDRESS", "IP_RANGE"] ip_range = scan.make_event("127.0.0.0/24", dummy=True) localhost4 = scan.make_event("127.0.0.1", source=ip_range) - localhost4.set_scope_distance(0) + localhost4.scope_distance = 0 localhost4.module = "plumbus" assert base_module._event_precheck(localhost4)[0] == True localhost4.module = "speculate" @@ -189,11 +189,11 @@ async def test_modules_basic_perhostonly(scan, helpers, events, bbot_config, bbo url_1 = per_host_scan.make_event( "http://evilcorp.com/1", event_type="URL", source=per_host_scan.root_event, tags=["status-200"] ) - url_1.set_scope_distance(0) + url_1.scope_distance = 0 url_2 = per_host_scan.make_event( "http://evilcorp.com/2", event_type="URL", source=per_host_scan.root_event, tags=["status-200"] ) - url_2.set_scope_distance(0) + url_2.scope_distance = 0 valid_1, reason_1 = await module._event_postcheck(url_1) valid_2, reason_2 = await module._event_postcheck(url_2) @@ -229,11 +229,11 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, b url_1 = per_domain_scan.make_event( "http://www.evilcorp.com/1", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"] ) - url_1.set_scope_distance(0) + url_1.scope_distance = 0 url_2 = per_domain_scan.make_event( "http://mail.evilcorp.com/2", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"] ) - url_2.set_scope_distance(0) + url_2.scope_distance = 0 valid_1, reason_1 = await module._event_postcheck(url_1) valid_2, reason_2 = await module._event_postcheck(url_2) diff --git a/pyproject.toml b/pyproject.toml index f8436f3bc0..b73db180ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ mkdocstrings-python = "^1.6.0" env = [ "BBOT_TESTING = True", "PYTHONASYNCIODEBUG = 1" -] +] [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] @@ -84,6 +84,7 @@ build-backend = "poetry_dynamic_versioning.backend" [tool.black] line-length = 119 +exclude = "bbot/test/test_step_1/test_manager_scope_shepherding.py" [tool.poetry-dynamic-versioning] enable = true